Examples with RebalancerState voldemort.server.rebalance.RebalancerState used on opensource projects

Example 11 with RebalancerState

use of voldemort.server.rebalance.RebalancerState in project voldemort by voldemort.

the class AdminRebalanceTest method testClusterAndRebalanceStateChange.

@Test(timeout = 60000)
public void testClusterAndRebalanceStateChange() throws IOException {
    try {
        startFourNodeRW();
        // Test 1) Normal case where-in all are up
        adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, false, true, true, true, true);
        List<Integer> nodesChecked = Lists.newArrayList();
        for (RebalanceTaskInfo plan : plans) {
            nodesChecked.add(plan.getStealerId());
            assertEquals(servers[plan.getStealerId()].getMetadataStore().getRebalancerState(), new RebalancerState(Lists.newArrayList(plan)));
            assertEquals(servers[plan.getStealerId()].getMetadataStore().getCluster(), finalCluster);
        }
        List<Integer> allNodes = Lists.newArrayList(Utils.nodeListToNodeIdList(Lists.newArrayList(currentCluster.getNodes())));
        allNodes.removeAll(nodesChecked);
        // Check all other nodes
        for (int nodeId : allNodes) {
            assertEquals(servers[nodeId].getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
            assertEquals(servers[nodeId].getMetadataStore().getCluster(), finalCluster);
        }
        // Clean-up everything
        cleanUpAllState();
        // Test 2) Add a plan before hand on one of them which should
        // trigger a rollback
        servers[3].getMetadataStore().getRebalancerState().update(new RebalanceTaskInfo(3, 0, new HashMap<String, List<Integer>>(), currentCluster));
        try {
            adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, false, true, true, true, true);
            fail("Should have thrown an exception since we added state before hand");
        } catch (VoldemortRebalancingException e) {
        }
        // except node 3 + all of them should have old cluster metadata
        for (VoldemortServer server : servers) {
            if (server.getMetadataStore().getNodeId() != 3) {
                assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
            }
            assertEquals(server.getMetadataStore().getCluster(), currentCluster);
        }
        // Clean-up everything
        cleanUpAllState();
        // Test 3) Shut one node down
        ServerTestUtils.stopVoldemortServer(servers[3]);
        servers[3] = null;
        try {
            adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, false, true, true, true, true);
            fail("Should have thrown an exception since we added state before hand");
        } catch (VoldemortRebalancingException e) {
        }
        // exception node 3
        for (VoldemortServer server : servers) {
            if (server != null) {
                assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
                assertEquals(server.getMetadataStore().getCluster(), currentCluster);
            }
        }
    } finally {
        shutDown();
    }
}

Example 12 with RebalancerState

use of voldemort.server.rebalance.RebalancerState in project voldemort by voldemort.

the class AdminRebalanceTest method testRebalanceNodeRO.

@Test(timeout = 60000)
public void testRebalanceNodeRO() throws IOException {
    try {
        startFourNodeRO();
        int numChunks = 5;
        for (StoreDefinition storeDef : Lists.newArrayList(storeDef1, storeDef2)) {
            buildROStore(storeDef, numChunks);
        }
        // Set into rebalancing state
        for (RebalanceTaskInfo partitionPlan : plans) {
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_SOURCE_CLUSTER_XML, partitionPlan.getInitialCluster());
        }
        // Actually run it
        try {
            for (RebalanceTaskInfo currentPlan : plans) {
                int asyncId = adminClient.rebalanceOps.rebalanceNode(currentPlan);
                assertNotSame("Got a valid rebalanceAsyncId", -1, asyncId);
                getAdminClient().rpcOps.waitForCompletion(currentPlan.getStealerId(), asyncId, 300, TimeUnit.SECONDS);
                // Test that plan has been removed from the list
                assertFalse(getServer(currentPlan.getStealerId()).getMetadataStore().getRebalancerState().getAll().contains(currentPlan));
            }
        } catch (Exception e) {
            e.printStackTrace();
            fail("Should not throw any exceptions");
        }
        // Check if files have been copied
        for (StoreDefinition storeDef : Lists.newArrayList(storeDef1, storeDef2)) {
            String storeName = storeDef.getName();
            for (RebalanceTaskInfo currentPlan : plans) {
                MetadataStore metadataStore = getServer(currentPlan.getStealerId()).getMetadataStore();
                int nodeId = metadataStore.getNodeId();
                int zoneId = metadataStore.getCluster().getNodeById(nodeId).getZoneId();
                StoreRoutingPlan storeRoutingPlan = new StoreRoutingPlan(metadataStore.getCluster(), storeDef);
                File currentDir = new File(((ReadOnlyStorageEngine) getStore(currentPlan.getStealerId(), storeName)).getCurrentDirPath());
                if (currentPlan.getPartitionStores().contains(storeDef.getName())) {
                    for (Integer partitionId : currentPlan.getStoreToPartitionIds().get(storeName)) {
                        int zoneNary = -1;
                        // it means we don't want to consider that partition.
                        try {
                            zoneNary = storeRoutingPlan.getZoneNaryForNodesPartition(zoneId, nodeId, partitionId);
                        } catch (VoldemortException ve) {
                            continue;
                        }
                        if (zoneNary < storeDef.getReplicationFactor()) {
                            for (int chunkId = 0; chunkId < numChunks; chunkId++) {
                                assertTrue(new File(currentDir, partitionId + "_" + zoneNary + "_" + chunkId + ".data").exists());
                                assertTrue(new File(currentDir, partitionId + "_" + zoneNary + "_" + chunkId + ".index").exists());
                            }
                        }
                    }
                }
            }
        }
        // All servers should be back to normal state
        for (VoldemortServer server : servers) {
            assertEquals(server.getMetadataStore().getRebalancerState(), new RebalancerState(new ArrayList<RebalanceTaskInfo>()));
            assertEquals(server.getMetadataStore().getServerStateUnlocked(), MetadataStore.VoldemortState.NORMAL_SERVER);
        }
        // Test the "cluster + swap" changes
        // Test 1) Fail some swap by adding a dummy store
        servers[2].getMetadataStore().put(MetadataStore.STORES_KEY, Lists.newArrayList(storeDef1, storeDef2, new StoreDefinitionBuilder().setName("test3").setType(ReadOnlyStorageConfiguration.TYPE_NAME).setKeySerializer(new SerializerDefinition("string")).setValueSerializer(new SerializerDefinition("string")).setRoutingPolicy(RoutingTier.CLIENT).setRoutingStrategyType(RoutingStrategyType.CONSISTENT_STRATEGY).setReplicationFactor(2).setPreferredReads(1).setRequiredReads(1).setPreferredWrites(1).setRequiredWrites(1).build()));
        try {
            adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, false, true, true);
            fail("Should have thrown an exception since one node doesn't have the store");
        } catch (VoldemortException e) {
        }
        servers[2].getMetadataStore().put(MetadataStore.STORES_KEY, Lists.newArrayList(storeDef1, storeDef2));
        // Test that all servers are still using the old cluster and have
        // swapped successfully
        checkRO(currentCluster);
        // Test 2) All passes scenario
        adminClient.rebalanceOps.rebalanceStateChange(currentCluster, finalCluster, servers[2].getMetadataStore().getStoreDefList(), servers[2].getMetadataStore().getStoreDefList(), plans, true, true, false, true, true);
        checkRO(finalCluster);
        // mmap-ed. Should fail...
        for (RebalanceTaskInfo partitionPlan : plans) {
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.SERVER_STATE_KEY, MetadataStore.VoldemortState.REBALANCING_MASTER_SERVER);
            getServer(partitionPlan.getStealerId()).getMetadataStore().put(MetadataStore.REBALANCING_STEAL_INFO, new RebalancerState(Lists.newArrayList(RebalanceTaskInfo.create(partitionPlan.toJsonString()))));
        }
        // Actually run it
        try {
            int asyncId = adminClient.rebalanceOps.rebalanceNode(plans.get(0));
            getAdminClient().rpcOps.waitForCompletion(plans.get(0).getStealerId(), asyncId, 300, TimeUnit.SECONDS);
            fail("Should throw an exception");
        } catch (Exception e) {
        }
    } finally {
        shutDown();
    }
}

Example 13 with RebalancerState

use of voldemort.server.rebalance.RebalancerState in project voldemort by voldemort.

the class MetadataStore method deleteRebalancingState.

/**
     * Delete the partition steal information from the rebalancer state
     * 
     * @param stealInfo The steal information to delete
     */
public void deleteRebalancingState(RebalanceTaskInfo stealInfo) {
    // acquire write lock
    writeLock.lock();
    try {
        RebalancerState rebalancerState = getRebalancerState();
        if (!rebalancerState.remove(stealInfo))
            throw new IllegalArgumentException("Couldn't find " + stealInfo + " in " + rebalancerState + " while deleting");
        if (rebalancerState.isEmpty()) {
            logger.debug("Cleaning all rebalancing state");
            cleanAllRebalancingState();
        } else {
            put(REBALANCING_STEAL_INFO, rebalancerState);
            initCache(REBALANCING_STEAL_INFO);
        }
    } finally {
        writeLock.unlock();
    }
}