Examples with SerializerDefinition voldemort.serialization.SerializerDefinition used on opensource projects

Example 26 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class GetallNodeReachTest method testGetallTouchOneZone.

@Test
public void testGetallTouchOneZone() throws Exception {
    cluster = getFourNodeClusterWithZones();
    HashMap<Integer, Integer> zoneReplicationFactor = new HashMap<Integer, Integer>();
    zoneReplicationFactor.put(0, 2);
    zoneReplicationFactor.put(1, 1);
    zoneReplicationFactor.put(2, 1);
    storeDef = new StoreDefinitionBuilder().setName("test").setType(InMemoryStorageConfiguration.TYPE_NAME).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(4).setZoneReplicationFactor(zoneReplicationFactor).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setPreferredReads(2).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).build();
    makeStore();
    Versioned<byte[]> v = Versioned.value("v".getBytes());
    subStores.get(0).put(TestUtils.toByteArray("k011_zone0_only"), v, null);
    subStores.get(1).put(TestUtils.toByteArray("k011_zone0_only"), v, null);
    subStores.get(2).put(TestUtils.toByteArray("k100_zone1_only"), v, null);
    /* test single key getall */
    List<ByteArray> keys011 = new ArrayList<ByteArray>();
    keys011.add(TestUtils.toByteArray("k011_zone0_only"));
    List<ByteArray> keys100 = new ArrayList<ByteArray>();
    keys100.add(TestUtils.toByteArray("k100_zone1_only"));
    assertEquals(2, store.getAll(keys011, null).get(TestUtils.toByteArray("k011_zone0_only")).size());
    assertFalse(store.getAll(keys100, null).containsKey(TestUtils.toByteArray("k100_zone1_only")));
    /* test multiple keys getall */
    List<ByteArray> keys = new ArrayList<ByteArray>();
    keys.add(TestUtils.toByteArray("k011_zone0_only"));
    keys.add(TestUtils.toByteArray("k100_zone1_only"));
    Map<ByteArray, List<Versioned<byte[]>>> result = store.getAll(keys, null);
    assertEquals(2, result.get(TestUtils.toByteArray("k011_zone0_only")).size());
    assertFalse(result.containsKey(TestUtils.toByteArray("k100_zone1_only")));
}

Example 27 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class RedirectingStoreTest method setUp.

@Before
public void setUp() throws IOException, InterruptedException {
    currentCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1 }, { 2, 3 }, {} });
    targetCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 2, Arrays.asList(0));
    this.primaryPartitionsMoved = Lists.newArrayList(0);
    this.secondaryPartitionsMoved = Lists.newArrayList(2, 3);
    this.storeDef = new StoreDefinitionBuilder().setName("test").setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    File tempStoreXml = new File(TestUtils.createTempDir(), "stores.xml");
    FileUtils.writeStringToFile(tempStoreXml, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(storeDef)));
    this.servers = new VoldemortServer[3];
    for (int nodeId = 0; nodeId < 3; nodeId++) {
        this.servers[nodeId] = startServer(nodeId, tempStoreXml.getAbsolutePath(), currentCluster);
    }
    // Start another node for only this unit test
    HashMap<ByteArray, byte[]> entrySet = ServerTestUtils.createRandomKeyValuePairs(100);
    SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(Lists.newArrayList("tcp://" + currentCluster.getNodeById(0).getHost() + ":" + currentCluster.getNodeById(0).getSocketPort())));
    StoreClient<Object, Object> storeClient = factory.getStoreClient("test");
    this.primaryEntriesMoved = Maps.newHashMap();
    this.secondaryEntriesMoved = Maps.newHashMap();
    this.proxyPutTestPrimaryEntries = Maps.newHashMap();
    this.proxyPutTestSecondaryEntries = Maps.newHashMap();
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, currentCluster);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
        storeClient.put(new String(entry.getKey().get()), new String(entry.getValue()));
        List<Integer> pList = strategy.getPartitionList(entry.getKey().get());
        if (primaryPartitionsMoved.contains(pList.get(0))) {
            primaryEntriesMoved.put(entry.getKey(), entry.getValue());
        } else if (secondaryPartitionsMoved.contains(pList.get(0))) {
            secondaryEntriesMoved.put(entry.getKey(), entry.getValue());
        }
    }
    // Sleep a while for the queries to go through...
    // Hope the 'God of perfect timing' is on our side
    Thread.sleep(500);
    // steal a few primary key-value pairs for testing proxy put logic
    int cnt = 0;
    for (Entry<ByteArray, byte[]> entry : primaryEntriesMoved.entrySet()) {
        if (cnt > 3)
            break;
        this.proxyPutTestPrimaryEntries.put(entry.getKey(), entry.getValue());
        cnt++;
    }
    for (ByteArray key : this.proxyPutTestPrimaryEntries.keySet()) {
        this.primaryEntriesMoved.remove(key);
    }
    assertTrue("Not enough primary entries", primaryEntriesMoved.size() > 1);
    // steal a few secondary key-value pairs for testing proxy put logic
    cnt = 0;
    for (Entry<ByteArray, byte[]> entry : secondaryEntriesMoved.entrySet()) {
        if (cnt > 3)
            break;
        this.proxyPutTestSecondaryEntries.put(entry.getKey(), entry.getValue());
        cnt++;
    }
    for (ByteArray key : this.proxyPutTestSecondaryEntries.keySet()) {
        this.secondaryEntriesMoved.remove(key);
    }
    assertTrue("Not enough secondary entries", primaryEntriesMoved.size() > 1);
    RebalanceBatchPlan RebalanceBatchPlan = new RebalanceBatchPlan(currentCluster, targetCluster, Lists.newArrayList(storeDef));
    List<RebalanceTaskInfo> plans = Lists.newArrayList(RebalanceBatchPlan.getBatchPlan());
    // Set into rebalancing state
    for (RebalanceTaskInfo partitionPlan : plans) {
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(partitionPlan)));
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, currentCluster);
        // update original storedefs
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_STORES_XML, Lists.newArrayList(storeDef));
    }
    // Update the cluster metadata on all three nodes
    for (VoldemortServer server : servers) {
        server.getMetadataStore().put(MetadataStore.CLUSTER_KEY, targetCluster);
    }
}

Example 28 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class AbstractStoreClientFactory method getRawStore.

@SuppressWarnings("unchecked")
public <K, V, T> Store<K, V, T> getRawStore(String storeName, InconsistencyResolver<Versioned<V>> resolver, String customStoresXml, String clusterXmlString, FailureDetector fd) {
    logger.info("Client zone-id [" + this.routedStoreConfig.getClientZoneId() + "] Attempting to get raw store [" + storeName + "] ");
    if (logger.isDebugEnabled()) {
        for (URI uri : bootstrapUrls) {
            logger.debug("Client Bootstrap url [" + uri + "]");
        }
    }
    // Get cluster and store metadata
    String clusterXml = clusterXmlString;
    if (clusterXml == null) {
        logger.debug("Fetching cluster.xml ...");
        clusterXml = bootstrapMetadataWithRetries(MetadataStore.CLUSTER_KEY, bootstrapUrls);
    }
    this.cluster = clusterMapper.readCluster(new StringReader(clusterXml), false);
    String storesXml = customStoresXml;
    if (storesXml == null) {
        String storesKey = storeName;
        if (config.isFetchAllStoresXmlInBootstrap()) {
            storesKey = MetadataStore.STORES_KEY;
        }
        if (logger.isDebugEnabled()) {
            logger.debug("Fetching store definition for Store " + storeName + " key " + storesKey);
        }
        storesXml = bootstrapMetadataWithRetries(storesKey, bootstrapUrls);
    }
    if (logger.isDebugEnabled()) {
        logger.debug("Obtained cluster metadata xml" + clusterXml);
        logger.debug("Obtained stores  metadata xml" + storesXml);
    }
    storeDefs = storeMapper.readStoreList(new StringReader(storesXml), false);
    StoreDefinition storeDef = null;
    for (StoreDefinition d : storeDefs) if (d.getName().equals(storeName))
        storeDef = d;
    if (storeDef == null) {
        logger.error("Bootstrap - unknown store: " + storeName);
        throw new BootstrapFailureException("Unknown store '" + storeName + "'.");
    }
    if (logger.isDebugEnabled()) {
        logger.debug(this.cluster.toString(true));
        logger.debug(storeDef.toString());
    }
    boolean repairReads = !storeDef.isView();
    // construct mapping
    Map<Integer, Store<ByteArray, byte[], byte[]>> clientMapping = Maps.newHashMap();
    Map<Integer, NonblockingStore> nonblockingStores = Maps.newHashMap();
    Map<Integer, NonblockingStore> nonblockingSlopStores = Maps.newHashMap();
    Map<Integer, Store<ByteArray, Slop, byte[]>> slopStores = null;
    if (storeDef.hasHintedHandoffStrategyType())
        slopStores = Maps.newHashMap();
    for (Node node : this.cluster.getNodes()) {
        Store<ByteArray, byte[], byte[]> store = getStore(storeDef.getName(), node.getHost(), getPort(node), this.requestFormatType);
        clientMapping.put(node.getId(), store);
        NonblockingStore nonblockingStore = routedStoreFactory.toNonblockingStore(store);
        nonblockingStores.put(node.getId(), nonblockingStore);
        if (slopStores != null) {
            Store<ByteArray, byte[], byte[]> rawSlopStore = getStore("slop", node.getHost(), getPort(node), this.requestFormatType);
            Store<ByteArray, Slop, byte[]> slopStore = SerializingStore.wrap(rawSlopStore, slopKeySerializer, slopValueSerializer, new IdentitySerializer());
            slopStores.put(node.getId(), slopStore);
            nonblockingSlopStores.put(node.getId(), routedStoreFactory.toNonblockingStore(rawSlopStore));
        }
    }
    /*
         * Check if we need to retrieve a reference to the failure detector. For
         * system stores - the FD reference would be passed in.
         */
    FailureDetector failureDetectorRef = fd;
    if (failureDetectorRef == null) {
        failureDetectorRef = getFailureDetector();
    } else {
        logger.debug("Using existing failure detector.");
    }
    this.routedStoreConfig.setRepairReads(repairReads);
    Store<ByteArray, byte[], byte[]> store = routedStoreFactory.create(this.cluster, storeDef, clientMapping, nonblockingStores, slopStores, nonblockingSlopStores, failureDetectorRef, this.routedStoreConfig);
    store = new LoggingStore(store);
    if (isJmxEnabled) {
        StatTrackingStore statStore = new StatTrackingStore(store, this.aggregateStats, this.cachedStoreStats);
        statStore.getStats().registerJmx(identifierString);
        store = statStore;
    }
    if (this.config.isEnableCompressionLayer()) {
        if (storeDef.getKeySerializer().hasCompression() || storeDef.getValueSerializer().hasCompression()) {
            store = new CompressingStore(store, getCompressionStrategy(storeDef.getKeySerializer()), getCompressionStrategy(storeDef.getValueSerializer()));
        }
    }
    /*
         * Initialize the finalstore object only once the store object itself is
         * wrapped by a StatrackingStore seems like the finalstore object is
         * redundant?
         */
    Store<K, V, T> finalStore = (Store<K, V, T>) store;
    if (this.config.isEnableSerializationLayer()) {
        Serializer<K> keySerializer = (Serializer<K>) serializerFactory.getSerializer(storeDef.getKeySerializer());
        Serializer<V> valueSerializer = (Serializer<V>) serializerFactory.getSerializer(storeDef.getValueSerializer());
        if (storeDef.isView() && (storeDef.getTransformsSerializer() == null))
            throw new SerializationException("Transforms serializer must be specified with a view ");
        Serializer<T> transformsSerializer = (Serializer<T>) serializerFactory.getSerializer(storeDef.getTransformsSerializer() != null ? storeDef.getTransformsSerializer() : new SerializerDefinition("identity"));
        finalStore = SerializingStore.wrap(store, keySerializer, valueSerializer, transformsSerializer);
    }
    // resolver (if they gave us one)
    if (this.config.isEnableInconsistencyResolvingLayer()) {
        InconsistencyResolver<Versioned<V>> secondaryResolver = resolver == null ? new TimeBasedInconsistencyResolver() : resolver;
        finalStore = new InconsistencyResolvingStore<K, V, T>(finalStore, new ChainedResolver<Versioned<V>>(new VectorClockInconsistencyResolver(), secondaryResolver));
    }
    return finalStore;
}

Example 29 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class StoreRoutingPlanPerf method getStoreDef.

// Construct zone-appropriate 3/2/2 store def.
public StoreDefinition getStoreDef(int numZones) {
    HashMap<Integer, Integer> zoneRep = new HashMap<Integer, Integer>();
    for (int zoneId = 0; zoneId < numZones; zoneId++) {
        zoneRep.put(zoneId, 3);
    }
    int repFactor = numZones * 3;
    StoreDefinition storeDef = new StoreDefinitionBuilder().setName("ZZ322").setType(BdbStorageConfiguration.TYPE_NAME).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setReplicationFactor(repFactor).setZoneReplicationFactor(zoneRep).setRequiredReads(2).setRequiredWrites(2).setZoneCountReads(0).setZoneCountWrites(0).build();
    return storeDef;
}

Example 30 with SerializerDefinition

use of voldemort.serialization.SerializerDefinition in project voldemort by voldemort.

the class AdminServiceBasicTest method testAddStore.

@Test
public void testAddStore() throws Exception {
    AdminClient adminClient = getAdminClient();
    doClientOperation();
    // Try to add a store whose replication factor is greater than the
    // number of nodes
    StoreDefinition definition = new StoreDefinitionBuilder().setName("updateTest").setType(InMemoryStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(3).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    try {
        adminClient.storeMgmtOps.addStore(definition);
        fail("Should have thrown an exception because we cannot add a store with a replication factor greater than number of nodes");
    } catch (Exception e) {
    }
    // Try adding a legit store using inmemorystorage engine
    definition = new StoreDefinitionBuilder().setName("updateTest").setType(InMemoryStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    adminClient.storeMgmtOps.addStore(definition);
    validateQuota(definition.getName());
    // now test the store
    StoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(cluster.getNodeById(0).getSocketUrl().toString()));
    StoreClient<Object, Object> client = factory.getStoreClient("updateTest");
    client.put("abc", "123");
    String s = (String) client.get("abc").getValue();
    assertEquals(s, "123");
    // test again with a unknown store
    try {
        client = factory.getStoreClient("updateTest2");
        client.put("abc", "123");
        s = (String) client.get("abc").getValue();
        assertEquals(s, "123");
        fail("Should have received bootstrap failure exception");
    } catch (Exception e) {
        if (!(e instanceof BootstrapFailureException))
            throw e;
    }
    // make sure that the store list we get back from AdminClient
    Versioned<List<StoreDefinition>> list = adminClient.metadataMgmtOps.getRemoteStoreDefList(0);
    assertTrue(list.getValue().contains(definition));
    // Now add a RO store
    definition = new StoreDefinitionBuilder().setName("addStoreROFormatTest").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(1).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    adminClient.storeMgmtOps.addStore(definition);
    validateQuota(definition.getName());
    // Retrieve list of read-only stores
    List<String> storeNames = Lists.newArrayList();
    for (StoreDefinition storeDef : adminClient.metadataMgmtOps.getRemoteStoreDefList(0).getValue()) {
        if (storeDef.getType().compareTo(ReadOnlyStorageConfiguration.TYPE_NAME) == 0) {
            storeNames.add(storeDef.getName());
        }
    }
    Map<String, String> storeToStorageFormat = adminClient.readonlyOps.getROStorageFormat(0, storeNames);
    for (String storeName : storeToStorageFormat.keySet()) {
        assertEquals(storeToStorageFormat.get(storeName), "ro2");
    }
    doClientOperation();
}