Examples with RoutingStrategy voldemort.routing.RoutingStrategy used on opensource projects

Example 16 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class RedirectingStoreTest method setUp.

@Before
public void setUp() throws IOException, InterruptedException {
    currentCluster = ServerTestUtils.getLocalCluster(3, new int[][] { { 0, 1 }, { 2, 3 }, {} });
    targetCluster = UpdateClusterUtils.createUpdatedCluster(currentCluster, 2, Arrays.asList(0));
    this.primaryPartitionsMoved = Lists.newArrayList(0);
    this.secondaryPartitionsMoved = Lists.newArrayList(2, 3);
    this.storeDef = new StoreDefinitionBuilder().setName("test").setType(BdbStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build();
    File tempStoreXml = new File(TestUtils.createTempDir(), "stores.xml");
    FileUtils.writeStringToFile(tempStoreXml, new StoreDefinitionsMapper().writeStoreList(Lists.newArrayList(storeDef)));
    this.servers = new VoldemortServer[3];
    for (int nodeId = 0; nodeId < 3; nodeId++) {
        this.servers[nodeId] = startServer(nodeId, tempStoreXml.getAbsolutePath(), currentCluster);
    }
    // Start another node for only this unit test
    HashMap<ByteArray, byte[]> entrySet = ServerTestUtils.createRandomKeyValuePairs(100);
    SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(Lists.newArrayList("tcp://" + currentCluster.getNodeById(0).getHost() + ":" + currentCluster.getNodeById(0).getSocketPort())));
    StoreClient<Object, Object> storeClient = factory.getStoreClient("test");
    this.primaryEntriesMoved = Maps.newHashMap();
    this.secondaryEntriesMoved = Maps.newHashMap();
    this.proxyPutTestPrimaryEntries = Maps.newHashMap();
    this.proxyPutTestSecondaryEntries = Maps.newHashMap();
    RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef, currentCluster);
    for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
        storeClient.put(new String(entry.getKey().get()), new String(entry.getValue()));
        List<Integer> pList = strategy.getPartitionList(entry.getKey().get());
        if (primaryPartitionsMoved.contains(pList.get(0))) {
            primaryEntriesMoved.put(entry.getKey(), entry.getValue());
        } else if (secondaryPartitionsMoved.contains(pList.get(0))) {
            secondaryEntriesMoved.put(entry.getKey(), entry.getValue());
        }
    }
    // Sleep a while for the queries to go through...
    // Hope the 'God of perfect timing' is on our side
    Thread.sleep(500);
    // steal a few primary key-value pairs for testing proxy put logic
    int cnt = 0;
    for (Entry<ByteArray, byte[]> entry : primaryEntriesMoved.entrySet()) {
        if (cnt > 3)
            break;
        this.proxyPutTestPrimaryEntries.put(entry.getKey(), entry.getValue());
        cnt++;
    }
    for (ByteArray key : this.proxyPutTestPrimaryEntries.keySet()) {
        this.primaryEntriesMoved.remove(key);
    }
    assertTrue("Not enough primary entries", primaryEntriesMoved.size() > 1);
    // steal a few secondary key-value pairs for testing proxy put logic
    cnt = 0;
    for (Entry<ByteArray, byte[]> entry : secondaryEntriesMoved.entrySet()) {
        if (cnt > 3)
            break;
        this.proxyPutTestSecondaryEntries.put(entry.getKey(), entry.getValue());
        cnt++;
    }
    for (ByteArray key : this.proxyPutTestSecondaryEntries.keySet()) {
        this.secondaryEntriesMoved.remove(key);
    }
    assertTrue("Not enough secondary entries", primaryEntriesMoved.size() > 1);
    RebalanceBatchPlan RebalanceBatchPlan = new RebalanceBatchPlan(currentCluster, targetCluster, Lists.newArrayList(storeDef));
    List<RebalanceTaskInfo> plans = Lists.newArrayList(RebalanceBatchPlan.getBatchPlan());
    // Set into rebalancing state
    for (RebalanceTaskInfo partitionPlan : plans) {
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(partitionPlan)));
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, currentCluster);
        // update original storedefs
        servers[partitionPlan.getStealerId()].getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_STORES_XML, Lists.newArrayList(storeDef));
    }
    // Update the cluster metadata on all three nodes
    for (VoldemortServer server : servers) {
        server.getMetadataStore().put(MetadataStore.CLUSTER_KEY, targetCluster);
    }
}

Example 17 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ConfigureNodesLocalHostTest method testConfigureNodesLocalHost.

/*
     * Checks to see that the local host is obtained as the first node in the
     * list returned by ConfigureNodesLocalHost
     */
@Test
public void testConfigureNodesLocalHost() throws Exception {
    List<Node> nodes = getTestNodes();
    Cluster cluster = new Cluster("test-route-all-local-pref-cluster", nodes);
    FailureDetector failureDetector = new ThresholdFailureDetector(new FailureDetectorConfig().setCluster(cluster));
    RoutingStrategy routingStrategy = new RouteToAllLocalPrefStrategy(cluster.getNodesShuffled());
    BasicPipelineData<byte[]> pipelineData = new BasicPipelineData<byte[]>();
    ConfigureNodesLocalHost<byte[], BasicPipelineData<byte[]>> action = new ConfigureNodesLocalHost<byte[], BasicPipelineData<byte[]>>(pipelineData, Event.COMPLETED, failureDetector, 1, routingStrategy, aKey);
    Pipeline pipeline = new Pipeline(Operation.GET, 10000, TimeUnit.MILLISECONDS);
    pipeline.addEventAction(Event.STARTED, action);
    pipeline.addEvent(Event.STARTED);
    pipeline.execute();
    if (pipelineData.getFatalError() != null)
        throw pipelineData.getFatalError();
    assertEquals(cluster.getNodes().size(), pipelineData.getNodes().size());
    assertEquals(cluster.getNodesShuffled().size(), pipelineData.getNodes().size());
    assertEquals(pipelineData.getNodes().get(0).getHost(), currentHost);
}

Example 18 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ConfigureNodesTest method testConfigureNodesWithZones.

@Test
public void testConfigureNodesWithZones() throws Exception {
    RoutingStrategy routingStrategy = new ZoneRoutingStrategy(clusterWithZones, storeDef.getZoneReplicationFactor(), storeDef.getReplicationFactor());
    BasicPipelineData<byte[]> pipelineData = new BasicPipelineData<byte[]>();
    Pipeline pipeline = new Pipeline(Operation.PUT, 10000, TimeUnit.MILLISECONDS);
    // PUT with changing zone id
    for (int clusterZoneId = 0; clusterZoneId < clusterWithZones.getNumberOfZones(); clusterZoneId++) {
        pipelineData.setZonesRequired(storeDef.getZoneCountWrites());
        pipeline.addEventAction(Event.STARTED, new ConfigureNodes<byte[], BasicPipelineData<byte[]>>(pipelineData, Event.COMPLETED, failureDetectorWithZones, storeDef.getRequiredReads(), routingStrategy, aKey, clusterWithZones.getZoneById(clusterZoneId)));
        pipeline.addEvent(Event.STARTED);
        pipeline.execute();
        List<Node> pipelineNodes = pipelineData.getNodes();
        int pipelineNodesIndex = 0;
        List<Integer> proximityList = clusterWithZones.getZoneById(clusterZoneId).getProximityList();
        // Check if returned list is as per the proximity list
        assertEquals(pipelineNodes.get(pipelineNodesIndex++).getZoneId(), clusterZoneId);
        for (; pipelineNodesIndex < pipelineNodes.size(); pipelineNodesIndex++) {
            assertEquals(proximityList.get(pipelineNodesIndex - 1), new Integer(pipelineNodes.get(pipelineNodesIndex).getZoneId()));
        }
    }
    // GET with changing zone requirements
    for (int zoneReq = 0; zoneReq < clusterWithZones.getNumberOfZones(); zoneReq++) {
        pipelineData = new BasicPipelineData<byte[]>();
        pipeline = new Pipeline(Operation.GET, 10000, TimeUnit.MILLISECONDS);
        pipelineData.setZonesRequired(zoneReq);
        pipeline.addEventAction(Event.STARTED, new ConfigureNodes<byte[], BasicPipelineData<byte[]>>(pipelineData, Event.COMPLETED, failureDetectorWithZones, storeDef.getRequiredReads(), routingStrategy, aKey, clusterWithZones.getZoneById(0)));
        pipeline.addEvent(Event.STARTED);
        pipeline.execute();
        // Check the first few nodes which are from different zones
        int zoneId = 1;
        for (int index = 0; index < zoneReq; index++) {
            assertEquals(pipelineData.getNodes().get(index).getZoneId(), zoneId++);
        }
    }
}

Example 19 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class GetAllConfigureNodesTest method testConfigureNodes.

@Test
public void testConfigureNodes() throws Exception {
    RoutingStrategy routingStrategy = new RouteToAllStrategy(cluster.getNodesShuffled());
    GetAllPipelineData pipelineData = new GetAllPipelineData();
    List<ByteArray> keys = new ArrayList<ByteArray>();
    for (int i = 0; i < 10; i++) keys.add(TestUtils.toByteArray("key-" + i));
    int preferred = cluster.getNumberOfNodes() - 1;
    GetAllConfigureNodes action = new GetAllConfigureNodes(pipelineData, Event.COMPLETED, failureDetector, preferred, preferred - 1, routingStrategy, keys, null, null, null);
    Pipeline pipeline = new Pipeline(Operation.GET, 10000, TimeUnit.MILLISECONDS);
    pipeline.addEventAction(Event.STARTED, action);
    pipeline.addEvent(Event.STARTED);
    pipeline.execute();
    if (pipelineData.getFatalError() != null)
        throw pipelineData.getFatalError();
    for (ByteArray key : keys) {
        List<Node> allNodesList = routingStrategy.routeRequest(key.get());
        assertEquals(cluster.getNumberOfNodes(), allNodesList.size());
        List<Node> extraNodes = pipelineData.getKeyToExtraNodesMap().get(key);
        assertEquals(cluster.getNumberOfNodes() - preferred, extraNodes.size());
        Node expectedExtraNode = allNodesList.get(preferred);
        Node actualExtraNode = extraNodes.get(0);
        assertEquals(expectedExtraNode, actualExtraNode);
        List<Node> preferredNodes = allNodesList.subList(0, preferred);
        assertEquals(preferred, preferredNodes.size());
        for (Node node : preferredNodes) {
            List<ByteArray> nodeKeys = pipelineData.getNodeToKeysMap().get(node);
            if (!nodeKeys.contains(key))
                fail();
        }
    }
}

Example 20 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ConsistencyCheckTest method testOnePartitionEndToEndBasedOnVersion.

@Test
public void testOnePartitionEndToEndBasedOnVersion() throws Exception {
    long now = System.currentTimeMillis();
    // setup four nodes with one store and one partition
    final SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2, 10000, 100000, 32 * 1024);
    VoldemortServer[] servers = new VoldemortServer[4];
    int[][] partitionMap = { { 0 }, { 1 }, { 2 }, { 3 } };
    Cluster cluster = ServerTestUtils.startVoldemortCluster(4, servers, partitionMap, socketStoreFactory, true, null, STORES_XML, new Properties());
    Node node = cluster.getNodeById(0);
    String bootstrapUrl = "tcp://" + node.getHost() + ":" + node.getSocketPort();
    AdminClient adminClient = new AdminClient(bootstrapUrl);
    byte[] value = { 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    byte[] value2 = { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9 };
    // make versions
    VectorClock vc1 = new VectorClock();
    VectorClock vc2 = new VectorClock();
    VectorClock vc3 = new VectorClock();
    // [0:1]
    vc1.incrementVersion(0, now);
    // [1:1]
    vc2.incrementVersion(1, now - 5000);
    // [0:1], over a day old
    vc3.incrementVersion(0, now - 89000000);
    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n0store = new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n1store = new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n2store = new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<Pair<ByteArray, Versioned<byte[]>>> n3store = new ArrayList<Pair<ByteArray, Versioned<byte[]>>>();
    ArrayList<ByteArray> keysHashedToPar0 = new ArrayList<ByteArray>();
    // find store
    Versioned<List<StoreDefinition>> storeDefinitions = adminClient.metadataMgmtOps.getRemoteStoreDefList(0);
    List<StoreDefinition> StoreDefinitions = storeDefinitions.getValue();
    StoreDefinition storeDefinition = null;
    for (StoreDefinition def : StoreDefinitions) {
        if (def.getName().equals(STORE_NAME)) {
            storeDefinition = def;
            break;
        }
    }
    assertNotNull("No such store found: " + STORE_NAME, storeDefinition);
    RoutingStrategy router = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
    while (keysHashedToPar0.size() < 7) {
        // generate random key
        Map<ByteArray, byte[]> map = ServerTestUtils.createRandomKeyValuePairs(1);
        ByteArray key = map.keySet().iterator().next();
        key.get()[0] = (byte) keysHashedToPar0.size();
        Integer masterPartition = router.getMasterPartition(key.get());
        if (masterPartition == 0) {
            keysHashedToPar0.add(key);
        } else {
            continue;
        }
    }
    ByteArray k6 = keysHashedToPar0.get(6);
    ByteArray k5 = keysHashedToPar0.get(5);
    ByteArray k4 = keysHashedToPar0.get(4);
    ByteArray k3 = keysHashedToPar0.get(3);
    ByteArray k2 = keysHashedToPar0.get(2);
    ByteArray k1 = keysHashedToPar0.get(1);
    ByteArray k0 = keysHashedToPar0.get(0);
    // insert K6 into node 0,1,2
    Versioned<byte[]> v6 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k6, v6));
    n1store.add(Pair.create(k6, v6));
    n2store.add(Pair.create(k6, v6));
    // insert K6(conflicting value and version) into node 0,1,2,3
    Versioned<byte[]> v6ConflictEarly = new Versioned<byte[]>(value2, vc2);
    n0store.add(Pair.create(k6, v6ConflictEarly));
    n1store.add(Pair.create(k6, v6ConflictEarly));
    n2store.add(Pair.create(k6, v6ConflictEarly));
    n3store.add(Pair.create(k6, v6ConflictEarly));
    // insert K4,K5 into four nodes
    Versioned<byte[]> v5 = new Versioned<byte[]>(value, vc1);
    Versioned<byte[]> v4 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k5, v5));
    n1store.add(Pair.create(k5, v5));
    n2store.add(Pair.create(k5, v5));
    n3store.add(Pair.create(k5, v5));
    n0store.add(Pair.create(k4, v4));
    n1store.add(Pair.create(k4, v4));
    n2store.add(Pair.create(k4, v4));
    n3store.add(Pair.create(k4, v4));
    // insert K3 into node 0,1,2
    Versioned<byte[]> v3 = new Versioned<byte[]>(value, vc2);
    n0store.add(Pair.create(k3, v3));
    n1store.add(Pair.create(k3, v3));
    n2store.add(Pair.create(k3, v3));
    // insert K3(conflicting but latest version) into node 0,1,2,3
    Versioned<byte[]> v3ConflictLate = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k3, v3ConflictLate));
    n1store.add(Pair.create(k3, v3ConflictLate));
    n2store.add(Pair.create(k3, v3ConflictLate));
    n3store.add(Pair.create(k3, v3ConflictLate));
    // insert K2 into node 0,1
    Versioned<byte[]> v2 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k2, v2));
    n1store.add(Pair.create(k2, v2));
    // insert K1 into node 0
    Versioned<byte[]> v1 = new Versioned<byte[]>(value, vc1);
    n0store.add(Pair.create(k1, v1));
    // insert K0(out of retention) into node 0,1,2
    Versioned<byte[]> v0 = new Versioned<byte[]>(value, vc3);
    n0store.add(Pair.create(k0, v0));
    n1store.add(Pair.create(k0, v0));
    n2store.add(Pair.create(k0, v0));
    // stream to store
    adminClient.streamingOps.updateEntries(0, STORE_NAME, n0store.iterator(), null);
    adminClient.streamingOps.updateEntries(1, STORE_NAME, n1store.iterator(), null);
    adminClient.streamingOps.updateEntries(2, STORE_NAME, n2store.iterator(), null);
    adminClient.streamingOps.updateEntries(3, STORE_NAME, n3store.iterator(), null);
    // should have FULL:2(K4,K5), LATEST_CONSISTENT:1(K3),
    // INCONSISTENT:2(K6,K2), ignored(K1,K0)
    List<String> urls = new ArrayList<String>();
    urls.add(bootstrapUrl);
    ConsistencyCheck.ComparisonType[] comparisonTypes = ConsistencyCheck.ComparisonType.values();
    for (ConsistencyCheck.ComparisonType type : comparisonTypes) {
        StringWriter sw = new StringWriter();
        ConsistencyCheck checker = new ConsistencyCheck(urls, STORE_NAME, 0, sw, type);
        Reporter reporter = null;
        checker.connect();
        reporter = checker.execute();
        assertEquals(7 - 2, reporter.numTotalKeys);
        assertEquals(3, reporter.numGoodKeys);
    }
    for (VoldemortServer vs : servers) {
        vs.stop();
    }
}