Examples with RoutingStrategy voldemort.routing.RoutingStrategy used on opensource projects

Example 26 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class HdfsFetcher method getPartitionsForCurrentNode.

/**
     * @return the set of partitions which need to be served (and thus fetched) by the current node.
     */
private Set<Integer> getPartitionsForCurrentNode(MetadataStore metadataStore, String storeName) {
    Set<Integer> partitions = Sets.newHashSet();
    StoreDefinition storeDefinition = metadataStore.getStoreDef(storeName);
    Cluster cluster = metadataStore.getCluster();
    List<Integer> partitionsMasteredByCurrentNode = cluster.getNodeById(metadataStore.getNodeId()).getPartitionIds();
    RoutingStrategy routingStrategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDefinition, cluster);
    // For each partition in the cluster, determine if it needs to be served by the current node
    for (int partitionId = 0; partitionId < cluster.getNumberOfPartitions(); partitionId++) {
        boolean partitionIdIsServedByCurrentNode = false;
        for (Integer replicatingPartition : routingStrategy.getReplicatingPartitionList(partitionId)) {
            if (partitionsMasteredByCurrentNode.contains(replicatingPartition)) {
                partitionIdIsServedByCurrentNode = true;
                break;
            }
        }
        if (partitionIdIsServedByCurrentNode) {
            partitions.add(partitionId);
        } else {
            logger.debug("Partition " + partitionId + " is not served by this node, so it will be skipped.");
        }
    }
    return partitions;
}

Example 27 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class AdminRebalanceTest method testRebalanceNodeRW2.

@Test(timeout = 60000)
public void testRebalanceNodeRW2() throws IOException {
    try {
        startFourNodeRW();
        // Start another node for only this unit test
        HashMap<ByteArray, byte[]> entrySet = ServerTestUtils.createRandomKeyValuePairs(TEST_SIZE);
        SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(Lists.newArrayList("tcp://" + currentCluster.getNodeById(0).getHost() + ":" + currentCluster.getNodeById(0).getSocketPort())));
        StoreClient<Object, Object> storeClient1 = factory.getStoreClient("test"), storeClient2 = factory.getStoreClient("test2");
        List<Integer> primaryPartitionsMoved = Lists.newArrayList(0);
        List<Integer> secondaryPartitionsMoved = Lists.newArrayList(8, 9, 10, 11);
        List<Integer> tertiaryPartitionsMoved = Lists.newArrayList(4, 5, 6, 7);
        HashMap<ByteArray, byte[]> primaryEntriesMoved = Maps.newHashMap();
        HashMap<ByteArray, byte[]> secondaryEntriesMoved = Maps.newHashMap();
        HashMap<ByteArray, byte[]> tertiaryEntriesMoved = Maps.newHashMap();
        RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef2, currentCluster);
        for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
            storeClient1.put(new String(entry.getKey().get()), new String(entry.getValue()));
            storeClient2.put(new String(entry.getKey().get()), new String(entry.getValue()));
            List<Integer> pList = strategy.getPartitionList(entry.getKey().get());
            if (primaryPartitionsMoved.contains(pList.get(0))) {
                primaryEntriesMoved.put(entry.getKey(), entry.getValue());
            } else if (secondaryPartitionsMoved.contains(pList.get(0))) {
                secondaryEntriesMoved.put(entry.getKey(), entry.getValue());
            } else if (tertiaryPartitionsMoved.contains(pList.get(0))) {
                tertiaryEntriesMoved.put(entry.getKey(), entry.getValue());
            }
        }
        // Set into rebalancing state
        for (RebalanceTaskInfo partitionPlan : plans) {
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, partitionPlan.getInitialCluster());
        }
        // Update the cluster metadata on all three nodes
        for (VoldemortServer server : servers) {
            server.getMetadataStore().put(MetadataStore.CLUSTER_KEY, finalCluster);
        }
        // Actually run it
        try {
            for (RebalanceTaskInfo currentPlan : plans) {
                int asyncId = adminClient.rebalanceOps.rebalanceNode(currentPlan);
                assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);
                getAdminClient().rpcOps.waitForCompletion(currentPlan.getStealerId(), asyncId, 300, TimeUnit.SECONDS);
                // Test that plan has been removed from the list
                assertFalse(getServer(currentPlan.getStealerId()).getMetadataStore().getRebalancerState().getAll().contains(currentPlan));
            }
        } catch (Exception e) {
            e.printStackTrace();
            fail("Should not throw any exceptions");
        }
        Store<ByteArray, byte[], byte[]> storeTest0 = getStore(0, "test2");
        Store<ByteArray, byte[], byte[]> storeTest1 = getStore(1, "test2");
        Store<ByteArray, byte[], byte[]> storeTest3 = getStore(3, "test2");
        Store<ByteArray, byte[], byte[]> storeTest00 = getStore(0, "test");
        Store<ByteArray, byte[], byte[]> storeTest30 = getStore(3, "test");
        // Primary
        for (Entry<ByteArray, byte[]> entry : primaryEntriesMoved.entrySet()) {
            // Test 2
            // Present on Node 0
            assertSame("entry should be present at store", 1, storeTest0.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest0.get(entry.getKey(), null).get(0).getValue()));
            // Present on Node 1
            assertSame("entry should be present at store", 1, storeTest1.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest1.get(entry.getKey(), null).get(0).getValue()));
            // Present on Node 3
            assertSame("entry should be present at store", 1, storeTest3.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest3.get(entry.getKey(), null).get(0).getValue()));
            // Test
            // Present on Node 0
            assertSame("entry should be present at store", 1, storeTest00.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest00.get(entry.getKey(), null).get(0).getValue()));
            // Present on Node 3
            assertSame("entry should be present at store", 1, storeTest30.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest30.get(entry.getKey(), null).get(0).getValue()));
        }
        // Secondary
        for (Entry<ByteArray, byte[]> entry : secondaryEntriesMoved.entrySet()) {
            // Test 2
            // Present on Node 0
            assertSame("entry should be present at store", 1, storeTest0.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest0.get(entry.getKey(), null).get(0).getValue()));
            // Present on Node 3
            assertSame("entry should be present at store", 1, storeTest3.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest3.get(entry.getKey(), null).get(0).getValue()));
            // Test
            // Present on Node 3
            assertSame("entry should be present at store", 1, storeTest30.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest30.get(entry.getKey(), null).get(0).getValue()));
        }
        // Tertiary
        for (Entry<ByteArray, byte[]> entry : tertiaryEntriesMoved.entrySet()) {
            // Test 2
            // Present on Node 3
            assertSame("entry should be present at store", 1, storeTest3.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest3.get(entry.getKey(), null).get(0).getValue()));
        }
        // All servers should be back to normal state
        for (VoldemortServer server : servers) {
            assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
            assertEquals(server.getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
        }
    } finally {
        shutDown();
    }
}

Example 28 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class AdminRebalanceTest method testRebalanceNodeRW.

@Test(timeout = 60000)
public void testRebalanceNodeRW() throws IOException {
    try {
        startThreeNodeRW();
        // Start another node for only this unit test
        HashMap<ByteArray, byte[]> entrySet = ServerTestUtils.createRandomKeyValuePairs(TEST_SIZE);
        SocketStoreClientFactory factory = new SocketStoreClientFactory(new ClientConfig().setBootstrapUrls(Lists.newArrayList("tcp://" + currentCluster.getNodeById(0).getHost() + ":" + currentCluster.getNodeById(0).getSocketPort())));
        StoreClient<Object, Object> storeClient1 = factory.getStoreClient("test"), storeClient2 = factory.getStoreClient("test2");
        List<Integer> primaryPartitionsMoved = Lists.newArrayList(0);
        List<Integer> secondaryPartitionsMoved = Lists.newArrayList(4, 5, 6, 7);
        HashMap<ByteArray, byte[]> primaryEntriesMoved = Maps.newHashMap();
        HashMap<ByteArray, byte[]> secondaryEntriesMoved = Maps.newHashMap();
        RoutingStrategy strategy = new RoutingStrategyFactory().updateRoutingStrategy(storeDef2, currentCluster);
        for (Entry<ByteArray, byte[]> entry : entrySet.entrySet()) {
            storeClient1.put(new String(entry.getKey().get()), new String(entry.getValue()));
            storeClient2.put(new String(entry.getKey().get()), new String(entry.getValue()));
            List<Integer> pList = strategy.getPartitionList(entry.getKey().get());
            if (primaryPartitionsMoved.contains(pList.get(0))) {
                primaryEntriesMoved.put(entry.getKey(), entry.getValue());
            } else if (secondaryPartitionsMoved.contains(pList.get(0))) {
                secondaryEntriesMoved.put(entry.getKey(), entry.getValue());
            }
        }
        try {
            adminClient.rebalanceOps.rebalanceNode(plans.get(0));
            fail("Should have thrown an exception since not in rebalancing state");
        } catch (VoldemortException e) {
        }
        // Set into rebalancing state
        for (RebalanceTaskInfo partitionPlan : plans) {
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, partitionPlan.getInitialCluster());
        }
        try {
            adminClient.rebalanceOps.rebalanceNode(plans.get(0));
            fail("Should have thrown an exception since no steal info");
        } catch (VoldemortException e) {
        }
        // Put a plan different from the plan that we actually want to
        // execute
        int incorrectStealerId = (plans.get(0).getStealerId() + 1) % 3;
        getServer(plans.get(0).getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(new RebalanceTaskInfo(incorrectStealerId, plans.get(0).getDonorId(), plans.get(0).getStoreToPartitionIds(), plans.get(0).getInitialCluster()))));
        try {
            adminClient.rebalanceOps.rebalanceNode(plans.get(0));
            fail("Should have thrown an exception since the two plans eventhough have the same donor are different");
        } catch (VoldemortException e) {
        }
        // Set the rebalance info on the stealer node
        for (RebalanceTaskInfo partitionPlan : plans) {
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
        }
        // Update the cluster metadata on all three nodes
        for (VoldemortServer server : servers) {
            server.getMetadataStore().put(MetadataStore.CLUSTER_KEY, finalCluster);
        }
        // Actually run it
        try {
            for (RebalanceTaskInfo currentPlan : plans) {
                int asyncId = adminClient.rebalanceOps.rebalanceNode(currentPlan);
                // AlreadyRebalancingException
                try {
                    adminClient.rebalanceOps.rebalanceNode(currentPlan);
                    fail("Should have thrown an exception since it is already rebalancing");
                } catch (AlreadyRebalancingException e) {
                }
                assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);
                getAdminClient().rpcOps.waitForCompletion(currentPlan.getStealerId(), asyncId, 300, TimeUnit.SECONDS);
                // Test that plan has been removed from the list
                assertFalse(getServer(currentPlan.getStealerId()).getMetadataStore().getRebalancerState().getAll().contains(currentPlan));
            }
        } catch (Exception e) {
            e.printStackTrace();
            fail("Should not throw any exceptions");
        }
        Store<ByteArray, byte[], byte[]> storeTest0 = getStore(0, "test2");
        Store<ByteArray, byte[], byte[]> storeTest2 = getStore(2, "test2");
        Store<ByteArray, byte[], byte[]> storeTest20 = getStore(2, "test");
        // Primary is on Node 0 and not on Node 1
        for (Entry<ByteArray, byte[]> entry : primaryEntriesMoved.entrySet()) {
            assertSame("entry should be present at store", 1, storeTest0.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest0.get(entry.getKey(), null).get(0).getValue()));
            // Check in other store
            assertSame("entry should be present in store test2 ", 1, storeTest20.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest20.get(entry.getKey(), null).get(0).getValue()));
        }
        // Secondary is on Node 2 and not on Node 0
        for (Entry<ByteArray, byte[]> entry : secondaryEntriesMoved.entrySet()) {
            assertSame("entry should be present at store", 1, storeTest2.get(entry.getKey(), null).size());
            assertEquals("entry value should match", new String(entry.getValue()), new String(storeTest2.get(entry.getKey(), null).get(0).getValue()));
        }
        // All servers should be back to normal state
        for (VoldemortServer server : servers) {
            assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
            assertEquals(server.getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
        }
    } finally {
        shutDown();
    }
}

Example 29 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class ZoneShrinkageEndToEndTest method generateRandomKey.

public static ByteArray generateRandomKey(Cluster cluster, Integer nodeId, Integer replicationFactor) {
    for (; ; ) {
        byte[] candidate = TestUtils.randomString("ABCDEFGHIJKLMN", 10).getBytes();
        RoutingStrategy rs = new ConsistentRoutingStrategy(cluster, replicationFactor);
        List<Node> routes = rs.routeRequest(candidate);
        if (routes.get(0).getId() == nodeId) {
            ByteArray key = new ByteArray(candidate);
            return key;
        }
    }
}

Example 30 with RoutingStrategy

use of voldemort.routing.RoutingStrategy in project voldemort by voldemort.

the class AbstractRebalanceTest method checkGetEntries.

protected void checkGetEntries(Node node, Cluster cluster, StoreDefinition def, Store<ByteArray, byte[], byte[]> store, HashMap<Integer, List<Integer>> flattenedPresentTuples, HashMap<String, String> baselineTuples, HashMap<String, VectorClock> baselineVersions) {
    RoutingStrategy routing = new RoutingStrategyFactory().updateRoutingStrategy(def, cluster);
    for (Entry<String, String> entry : baselineTuples.entrySet()) {
        ByteArray keyBytes = new ByteArray(ByteUtils.getBytes(entry.getKey(), "UTF-8"));
        List<Integer> partitions = routing.getPartitionList(keyBytes.get());
        if (StoreRoutingPlan.checkKeyBelongsToPartition(partitions, node.getPartitionIds(), flattenedPresentTuples)) {
            List<Versioned<byte[]>> values = store.get(keyBytes, null);
            // expecting exactly one version
            if (values.size() == 0) {
                fail("unable to find value for key=" + entry.getKey() + " on node=" + node.getId());
            }
            assertEquals("Expecting exactly one version", 1, values.size());
            Versioned<byte[]> value = values.get(0);
            // check version matches
            if (baselineVersions == null) {
                // expecting base version for all
                assertEquals("Value version should match", new VectorClock(), value.getVersion());
            } else {
                assertEquals("Value version should match", baselineVersions.get(entry.getKey()), value.getVersion());
            }
            // check value matches.
            assertEquals("Value bytes should match", entry.getValue(), ByteUtils.getString(value.getValue(), "UTF-8"));
        }
    }
}