Examples with StoreDefinitionBuilder voldemort.store.StoreDefinitionBuilder used on opensource projects

Example 16 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class RebalanceMetadataConsistencyTest method setUp.

@Before
public void setUp() {
    currentCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0 }, { 1, 3 }, { 2 } });
    finalCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0 }, { 1 }, { 2 }, { 3 } });
    rwStoreDefWithReplication = new StoreDefinitionBuilder().setName(testStoreNameRW).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    Store<String, String, String> innerStore = new InMemoryStorageEngine<String, String, String>("inner-store");
    innerStore.put(MetadataStore.CLUSTER_KEY, new Versioned<String>(new ClusterMapper().writeCluster(currentCluster)), null);
    innerStore.put(MetadataStore.STORES_KEY, new Versioned<String>(new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(rwStoreDefWithReplication))), null);
    rwStoreDefWithReplication2 = new StoreDefinitionBuilder().setName(testStoreNameRW2).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    metadataStore = MetadataStore.createInMemoryMetadataStore(innerStore, 0);
    rebalancer = new Rebalancer(null, metadataStore, null, null);
}

Example 17 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class StreamingClientTest method configs.

@Parameterized.Parameters
public static Collection<Object[]> configs() {
    StoreDefinition storeDefConsistentStrategy = new StoreDefinitionBuilder().setName(TEST_STORE_NAME).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(2).setRequiredWrites(2).build();
    HashMap<Integer, Integer> zoneReplicationFactor = new HashMap<Integer, Integer>();
    zoneReplicationFactor.put(1, 2);
    zoneReplicationFactor.put(3, 2);
    StoreDefinition storeDefZoneStrategy = new StoreDefinitionBuilder().setName(TEST_STORE_NAME).setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.ZONE_STRATEGY).setReplicationFactor(4).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).setZoneCountReads(0).setZoneCountWrites(0).setZoneReplicationFactor(zoneReplicationFactor).setHintedHandoffStrategy(HintedHandoffStrategyType.PROXIMITY_STRATEGY).build();
    return Arrays.asList(new Object[][] { { 2, ServerTestUtils.getLocalCluster(2, new int[][] { { 0, 1, 2, 3 }, { 4, 5, 6, 7 } }), 0, storeDefConsistentStrategy }, { 2, ServerTestUtils.getLocalNonContiguousNodesCluster(new int[] { 1, 3 }, new int[][] { { 0, 1, 2, 3 }, { 4, 5, 6, 7 } }), 1, storeDefConsistentStrategy }, { 6, ClusterTestUtils.getZ1Z3ClusterWithNonContiguousNodeIds(), 3, storeDefZoneStrategy } });
}

Example 18 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class AdminRebalanceTest method startFourNodeRORW.

public void startFourNodeRORW() throws IOException {
    storeDef1 = new StoreDefinitionBuilder().setName("test").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    storeDef2 = new StoreDefinitionBuilder().setName("test2").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(3).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    storeDef3 = ServerTestUtils.getStoreDef("test3", 2, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    storeDef4 = ServerTestUtils.getStoreDef("test4", 3, 1, 1, 1, 1, RoutingStrategyType.CONSISTENT_STRATEGY);
    File tempStoreXml = new File(TestUtils.createTempDir(), "stores.xml");
    FileUtils.writeStringToFile(tempStoreXml, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(storeDef1, storeDef2, storeDef3, storeDef4)));
    int numServers = 4;
    servers = new VoldemortServer[numServers];
    int[][] partitionMap = { { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10, 11 }, {} };
    currentCluster = ServerTestUtils.startVoldemortCluster(numServers, servers, partitionMap, socketStoreFactory, useNio, null, tempStoreXml.getAbsolutePath(), new Properties());
    finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 3, Lists.newArrayList(0));
    // Make plan only with RO stores
    RebalanceBatchPlan plan = new RebalanceBatchPlan(currentCluster, finalCluster, Lists.newArrayList(storeDef1, storeDef2));
    plans = plan.getBatchPlan();
    adminClient = ServerTestUtils.getAdminClient(currentCluster);
}

Example 19 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class AdminRebalanceTest method startFourNodeRO.

public void startFourNodeRO() throws IOException {
    storeDef1 = new StoreDefinitionBuilder().setName("test").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    storeDef2 = new StoreDefinitionBuilder().setName("test2").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(3).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    File tempStoreXml = new File(TestUtils.createTempDir(), "stores.xml");
    FileUtils.writeStringToFile(tempStoreXml, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(storeDef1, storeDef2)));
    int numServers = 4;
    servers = new VoldemortServer[numServers];
    int[][] partitionMap = { { 0, 1, 2, 3 }, { 4, 5, 6, 7 }, { 8, 9, 10, 11 }, {} };
    currentCluster = ServerTestUtils.startVoldemortCluster(numServers, servers, partitionMap, socketStoreFactory, useNio, null, tempStoreXml.getAbsolutePath(), new Properties());
    finalCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 3, Lists.newArrayList(0));
    RebalanceBatchPlan plan = new RebalanceBatchPlan(currentCluster, finalCluster, Lists.newArrayList(storeDef1, storeDef2));
    plans = Lists.newArrayList(plan.getBatchPlan());
    adminClient = ServerTestUtils.getAdminClient(currentCluster);
}

Example 20 with StoreDefinitionBuilder

use of voldemort.store.StoreDefinitionBuilder in project voldemort by voldemort.

the class AdminRebalanceTest method testRebalanceNodeRORW.

@Test(timeout = 60000)
public void testRebalanceNodeRORW() throws IOException, InterruptedException {
    try {
        startFourNodeRORW();
        int numChunks = 5;
        for (StoreDefinition storeDef : Lists.newArrayList(storeDef1, storeDef2)) {
            buildROStore(storeDef, numChunks);
        }
        // Set into rebalancing state
        for (RebalanceTaskInfo partitionPlan : plans) {
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, partitionPlan.getInitialCluster());
        }
        // Actually run it
        try {
            for (RebalanceTaskInfo currentPlan : plans) {
                int asyncId = adminClient.rebalanceOps.rebalanceNode(currentPlan);
                assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);
                getAdminClient().rpcOps.waitForCompletion(currentPlan.getStealerId(), asyncId, 300, TimeUnit.SECONDS);
                // Test that plan has been removed from the list
                assertFalse(getServer(currentPlan.getStealerId()).getMetadataStore().getRebalancerState().getAll().contains(currentPlan));
            }
        } catch (Exception e) {
            e.printStackTrace();
            fail("Should not throw any exceptions");
        }
        // Test 1) Change one of the rebalance partitions info to force a
        // failure
        servers[3].getMetadataStore().getRebalancerState().update(new RebalanceTaskInfo(3, 0, new HashMap<String, List<Integer>>(), currentCluster));
        try {
            adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, true, true, true);
            fail("Should have thrown an exception since we added state before hand");
        } catch (VoldemortRebalancingException e) {
        }
        // except node 3
        for (VoldemortServer server : servers) {
            if (server.getMetadataStore().getNodeId() != 3) {
                assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
                assertEquals(server.getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
            }
            assertEquals(server.getMetadataStore().getCluster(), currentCluster);
        }
        checkRO(currentCluster);
        // Clean-up everything
        cleanUpAllState();
        // Test 2 ) Add another store to trigger a failure
        servers[2].getMetadataStore().put(MetadataStore.STORES_KEY, Lists.newArrayList(storeDef1, storeDef2, storeDef3, storeDef4, new StoreDefinitionBuilder().setName("test5").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build()));
        try {
            adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, true, true, true);
            fail("Should have thrown an exception since we added state before hand");
        } catch (VoldemortRebalancingException e) {
        }
        Thread.sleep(1000);
        for (VoldemortServer server : servers) {
            assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
            assertEquals(server.getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
            assertEquals(server.getMetadataStore().getCluster(), currentCluster);
        }
        checkRO(currentCluster);
        // Clean-up everything
        cleanUpAllState();
        // Put back server 2 back to normal state
        servers[2].getMetadataStore().put(MetadataStore.STORES_KEY, Lists.newArrayList(storeDef1, storeDef2, storeDef3, storeDef4));
        // Test 3) Everything should work
        adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, true, true, true);
        List<Integer> nodesChecked = Lists.newArrayList();
        for (RebalanceTaskInfo plan : plans) {
            nodesChecked.add(plan.getStealerId());
            assertEquals(servers[plan.getStealerId()].getMetadataStore().getRebalancerState(), new RebalancerState(Lists.newArrayList(plan)));
            assertEquals(servers[plan.getStealerId()].getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            assertEquals(servers[plan.getStealerId()].getMetadataStore().getCluster(), finalCluster);
        }
        List<Integer> allNodes = Lists.newArrayList(Utils.nodeListToNodeIdList(Lists.newArrayList(currentCluster.getNodes())));
        allNodes.removeAll(nodesChecked);
        // Check all other nodes
        for (int nodeId : allNodes) {
            assertEquals(servers[nodeId].getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
            assertEquals(servers[nodeId].getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
            assertEquals(servers[nodeId].getMetadataStore().getCluster(), finalCluster);
        }
        checkRO(finalCluster);
    } finally {
        shutDown();
    }
}