Examples with RebalancePlan voldemort.client.rebalance.RebalancePlan used on opensource projects

Example 1 with RebalancePlan

use of voldemort.client.rebalance.RebalancePlan in project voldemort by voldemort.

the class RebalanceControllerCLI method main.

public static void main(String[] args) throws Exception {
    setupParser();
    OptionSet options = getValidOptions(args);
    // Bootstrap & fetch current cluster/stores
    String bootstrapURL = (String) options.valueOf("url");
    int parallelism = RebalanceController.MAX_PARALLEL_REBALANCING;
    if (options.has("parallelism")) {
        parallelism = (Integer) options.valueOf("parallelism");
    }
    long proxyPauseSec = RebalanceController.PROXY_PAUSE_IN_SECONDS;
    if (options.has("proxy-pause")) {
        proxyPauseSec = (Long) options.valueOf("proxy-pause");
    }
    RebalanceController rebalanceController = new RebalanceController(bootstrapURL, parallelism, proxyPauseSec);
    Cluster currentCluster = rebalanceController.getCurrentCluster();
    List<StoreDefinition> currentStoreDefs = rebalanceController.getCurrentStoreDefs();
    // If this test doesn't pass, something is wrong in prod!
    RebalanceUtils.validateClusterStores(currentCluster, currentStoreDefs);
    // Determine final cluster/stores and validate them
    String finalClusterXML = (String) options.valueOf("final-cluster");
    Cluster finalCluster = new ClusterMapper().readCluster(new File(finalClusterXML));
    List<StoreDefinition> finalStoreDefs = currentStoreDefs;
    if (options.has("final-stores")) {
        String storesXML = (String) options.valueOf("final-stores");
        finalStoreDefs = new StoreDefinitionsMapper().readStoreList(new File(storesXML));
    }
    RebalanceUtils.validateClusterStores(finalCluster, finalStoreDefs);
    RebalanceUtils.validateCurrentFinalCluster(currentCluster, finalCluster);
    // Process optional "planning" arguments
    int batchSize = CmdUtils.valueOf(options, "batch-size", RebalancePlan.BATCH_SIZE);
    String outputDir = null;
    if (options.has("output-dir")) {
        outputDir = (String) options.valueOf("output-dir");
    }
    RebalancePlan rebalancePlan = new RebalancePlan(currentCluster, currentStoreDefs, finalCluster, finalStoreDefs, batchSize, outputDir);
    boolean resetQuota = !options.has("no-reset-quota");
    Set<String> storeNames = Sets.newHashSet();
    for (StoreDefinition storeDef : finalStoreDefs) {
        storeNames.add(storeDef.getName());
    }
    QuotaResetter quotaResetter = new QuotaResetter(bootstrapURL, storeNames, rebalancePlan.getFinalCluster().getNodeIds());
    // before rebalance, remember and disable quota enforcement settings
    if (resetQuota) {
        quotaResetter.rememberAndDisableQuota();
    }
    // Plan & execute rebalancing.
    rebalanceController.rebalance(rebalancePlan);
    // after rebalance, reset quota values and recover quota enforcement
    if (resetQuota) {
        quotaResetter.resetQuotaAndRecoverEnforcement();
    }
}

Example 2 with RebalancePlan

use of voldemort.client.rebalance.RebalancePlan in project voldemort by voldemort.

the class RebalancePlanCLI method main.

public static void main(String[] args) throws Exception {
    setupParser();
    OptionSet options = getValidOptions(args);
    // Required args
    String currentClusterXML = (String) options.valueOf("current-cluster");
    String currentStoresXML = (String) options.valueOf("current-stores");
    String finalClusterXML = (String) options.valueOf("final-cluster");
    // Required args for some use cases
    String finalStoresXML = new String(currentStoresXML);
    if (options.has("final-stores")) {
        finalStoresXML = (String) options.valueOf("final-stores");
    }
    Cluster currentCluster = new ClusterMapper().readCluster(new File(currentClusterXML));
    List<StoreDefinition> currentStoreDefs = new StoreDefinitionsMapper().readStoreList(new File(currentStoresXML));
    Cluster finalCluster = new ClusterMapper().readCluster(new File(finalClusterXML));
    List<StoreDefinition> finalStoreDefs = new StoreDefinitionsMapper().readStoreList(new File(finalStoresXML));
    // Optional args
    int batchSize = CmdUtils.valueOf(options, "batch-size", RebalancePlan.BATCH_SIZE);
    String outputDir = null;
    if (options.has("output-dir")) {
        outputDir = (String) options.valueOf("output-dir");
    }
    new RebalancePlan(currentCluster, currentStoreDefs, finalCluster, finalStoreDefs, batchSize, outputDir);
}

Example 3 with RebalancePlan

use of voldemort.client.rebalance.RebalancePlan in project voldemort by voldemort.

the class ClusterTestUtils method getRebalanceKit.

public static RebalanceKit getRebalanceKit(String bootstrapUrl, Cluster finalCluster, List<StoreDefinition> finalStoreDefs) {
    RebalanceController rebalanceController = new RebalanceController(bootstrapUrl, RebalanceController.MAX_PARALLEL_REBALANCING, REBALANCE_CONTROLLER_TEST_PROXY_PAUSE_IN_SECONDS);
    RebalancePlan rebalancePlan = rebalanceController.getPlan(finalCluster, finalStoreDefs, RebalancePlan.BATCH_SIZE);
    return new RebalanceKit(rebalanceController, rebalancePlan);
}

Example 4 with RebalancePlan

use of voldemort.client.rebalance.RebalancePlan in project voldemort by voldemort.

the class ZoneCLipperTest method testDropZoneIdWorks.

@Test
public void testDropZoneIdWorks() {
    // Create a list of current partition ids. We will use this set to
    // compare partitions ids with the interim cluster
    Set<Integer> originalPartitions = new HashSet<Integer>();
    for (Integer zoneId : initialCluster.getZoneIds()) {
        originalPartitions.addAll(initialCluster.getPartitionIdsInZone(zoneId));
    }
    // Get an intermediate cluster where partitions that belong to the zone
    // that is being dropped have been moved to the existing zones
    Cluster interimCluster = RebalanceUtils.vacateZone(initialCluster, dropZoneId);
    // Make sure that the intermediate cluster should have same number of
    // partitions
    RebalanceUtils.validateClusterPartitionCounts(initialCluster, interimCluster);
    // Make sure that the intermediate cluster should have same number of
    // nodes
    RebalanceUtils.validateClusterNodeCounts(initialCluster, interimCluster);
    // Make sure that the intermediate cluster doesn't have any partitions in
    // the dropped zone
    assertTrue("Zone being dropped has partitions. ZoneClipper didn't work properly", interimCluster.getPartitionIdsInZone(dropZoneId).isEmpty());
    // Make sure that the nodes being dropped don't host any partitions
    for (Integer nodeId : interimCluster.getNodeIdsInZone(dropZoneId)) {
        assertTrue("Nodes in the zone being dropped don't have empty partitions list", interimCluster.getNodeById(nodeId).getPartitionIds().isEmpty());
    }
    Set<Integer> finalPartitions = new HashSet<Integer>();
    for (Integer zoneId : interimCluster.getZoneIds()) {
        finalPartitions.addAll(interimCluster.getPartitionIdsInZone(zoneId));
    }
    // Compare to original partition ids list
    assertTrue("Original and interm partition ids don't match", originalPartitions.equals(finalPartitions));
    // Make sure that there is no data movement
    RebalancePlan rebalancePlan = ClusterTestUtils.makePlan(initialCluster, storeDefs, interimCluster, storeDefs);
    // Make sure we have a plan
    assertEquals(rebalancePlan.getPlan().size(), 1);
    // Make sure there is no cross zones between zones in the plan
    assertEquals(rebalancePlan.getPartitionStoresMovedXZone(), 0);
    // Make sure there is no data movement between nodes
    assertEquals(rebalancePlan.getPartitionStoresMoved(), 0);
    for (Integer nodeId : interimCluster.getNodeIds()) {
        Set<Integer> remainingNodes = Sets.symmetricDifference(interimCluster.getNodeIds(), Sets.newHashSet(nodeId));
        for (Integer otherNodeId : remainingNodes) {
            assertTrue("Something went wrong as there is data movement between nodes", rebalancePlan.getNodeMoveMap().get(nodeId, otherNodeId) == 0);
        }
    }
    // Also get the adjusted store definitions, with the zone dropped
    Cluster finalCluster = RebalanceUtils.dropZone(interimCluster, dropZoneId);
    List<StoreDefinition> clippedStoreDefs = RebalanceUtils.dropZone(storeDefs, dropZoneId);
    for (StoreDefinition storeDef : clippedStoreDefs) {
        StoreDefinition orgStoreDef = StoreDefinitionUtils.getStoreDefinitionWithName(storeDefs, storeDef.getName());
        assertFalse("Clipped storedef has replication for dropped zone", storeDef.getZoneReplicationFactor().containsKey(dropZoneId));
        assertEquals("Clipped storedef has incorrect number of zones", initialCluster.getZones().size() - 1, storeDef.getZoneReplicationFactor().size());
        assertEquals("Clipped storedef has incorrect total repfactor", orgStoreDef.getReplicationFactor() - orgStoreDef.getZoneReplicationFactor().get(dropZoneId), storeDef.getReplicationFactor());
    }
    // Confirm that we would not route to any of the dropped nodes for any
    // store.
    Set<Integer> dropNodes = interimCluster.getNodeIdsInZone(dropZoneId);
    for (StoreDefinition storeDef : clippedStoreDefs) {
        StoreRoutingPlan routingPlan = new StoreRoutingPlan(finalCluster, storeDef);
        Node[] partitionToNode = finalCluster.getPartitionIdToNodeArray();
        for (int p = 0; p < partitionToNode.length; p++) {
            List<Integer> replicaNodes = routingPlan.getReplicationNodeList(p);
            assertFalse("Should not be routing to any dropped nodes", replicaNodes.removeAll(dropNodes));
        }
    }
}

Example 5 with RebalancePlan

use of voldemort.client.rebalance.RebalancePlan in project voldemort by voldemort.

the class ClusterTestUtils method getRebalanceKit.

public static RebalanceKit getRebalanceKit(String bootstrapUrl, int maxParallel, long proxyPauseS, Cluster finalCluster) {
    RebalanceController rebalanceController = new RebalanceController(bootstrapUrl, maxParallel, proxyPauseS);
    RebalancePlan rebalancePlan = rebalanceController.getPlan(finalCluster, RebalancePlan.BATCH_SIZE);
    return new RebalanceKit(rebalanceController, rebalancePlan);
}