Examples with AutoRebalanceStrategy org.apache.helix.controller.rebalancer.strategy.AutoRebalanceStrategy used on opensource projects

Example 1 with AutoRebalanceStrategy

use of org.apache.helix.controller.rebalancer.strategy.AutoRebalanceStrategy in project helix by apache.

the class TestAutoRebalanceStrategy method testWontMoveSinglePartitionUnnecessarily.

/**
 * Tests the following scenario: there is only a single partition for a resource. Two nodes up,
 * partition should
 * be assigned to one of them. Take down that node, partition should move. Bring back up that
 * node, partition should not move unnecessarily.
 */
@Test
public void testWontMoveSinglePartitionUnnecessarily() {
    final String RESOURCE = "resource";
    final String partition = "resource_0";
    final StateModelDefinition STATE_MODEL = new StateModelDefinition(StateModelConfigGenerator.generateConfigForOnlineOffline());
    LinkedHashMap<String, Integer> stateCount = Maps.newLinkedHashMap();
    stateCount.put("ONLINE", 1);
    final String[] NODES = { "n0", "n1" };
    // initial state, one node, no mapping
    List<String> allNodes = Lists.newArrayList(NODES);
    List<String> liveNodes = Lists.newArrayList(NODES);
    Map<String, Map<String, String>> currentMapping = Maps.newHashMap();
    currentMapping.put(partition, new HashMap<String, String>());
    // Both nodes there
    List<String> partitions = Lists.newArrayList(partition);
    Map<String, String> upperBounds = Maps.newHashMap();
    for (String state : STATE_MODEL.getStatesPriorityList()) {
        upperBounds.put(state, STATE_MODEL.getNumInstancesPerState(state));
    }
    ZNRecord znRecord = new AutoRebalanceStrategy(RESOURCE, partitions, stateCount, Integer.MAX_VALUE).computePartitionAssignment(allNodes, liveNodes, currentMapping, null);
    Map<String, List<String>> preferenceLists = znRecord.getListFields();
    List<String> preferenceList = preferenceLists.get(partition.toString());
    Assert.assertNotNull(preferenceList, "invalid preference list for " + partition);
    Assert.assertEquals(preferenceList.size(), 1, "invalid preference list for " + partition);
    String state = znRecord.getMapField(partition.toString()).get(preferenceList.get(0));
    Assert.assertEquals(state, "ONLINE", "Invalid state for " + partition);
    String preferredNode = preferenceList.get(0);
    String otherNode = preferredNode.equals(NODES[0]) ? NODES[1] : NODES[0];
    // ok, see what happens if we've got the partition on the other node (e.g. due to the preferred
    // node being down).
    currentMapping.get(partition).put(otherNode, state);
    znRecord = new AutoRebalanceStrategy(RESOURCE, partitions, stateCount, Integer.MAX_VALUE).computePartitionAssignment(allNodes, liveNodes, currentMapping, null);
    preferenceLists = znRecord.getListFields();
    preferenceList = preferenceLists.get(partition.toString());
    Assert.assertNotNull(preferenceList, "invalid preference list for " + partition);
    Assert.assertEquals(preferenceList.size(), 1, "invalid preference list for " + partition);
    state = znRecord.getMapField(partition.toString()).get(preferenceList.get(0));
    Assert.assertEquals(state, "ONLINE", "Invalid state for " + partition);
    String finalPreferredNode = preferenceList.get(0);
    // finally, make sure we haven't moved it.
    Assert.assertEquals(finalPreferredNode, otherNode);
}

Example 2 with AutoRebalanceStrategy

use of org.apache.helix.controller.rebalancer.strategy.AutoRebalanceStrategy in project helix by apache.

the class TestAutoRebalanceStrategy method testOrphansNotPreferred.

/**
 * Tests the following scenario: nodes come up one by one, then one node is taken down. Preference
 * lists should prefer nodes in the current mapping at all times, but when all nodes are in the
 * current mapping, then it should distribute states as evenly as possible.
 */
@Test
public void testOrphansNotPreferred() {
    final String RESOURCE_NAME = "resource";
    final String[] PARTITIONS = { "resource_0", "resource_1", "resource_2" };
    final StateModelDefinition STATE_MODEL = new StateModelDefinition(StateModelConfigGenerator.generateConfigForMasterSlave());
    final int REPLICA_COUNT = 2;
    final String[] NODES = { "n0", "n1", "n2" };
    // initial state, one node, no mapping
    List<String> allNodes = Lists.newArrayList(NODES[0]);
    List<String> liveNodes = Lists.newArrayList(NODES[0]);
    Map<String, Map<String, String>> currentMapping = Maps.newHashMap();
    for (String partition : PARTITIONS) {
        currentMapping.put(partition, new HashMap<String, String>());
    }
    // make sure that when the first node joins, a single replica is assigned fairly
    List<String> partitions = ImmutableList.copyOf(PARTITIONS);
    LinkedHashMap<String, Integer> stateCount = STATE_MODEL.getStateCountMap(liveNodes.size(), REPLICA_COUNT);
    ZNRecord znRecord = new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount).computePartitionAssignment(allNodes, liveNodes, currentMapping, null);
    Map<String, List<String>> preferenceLists = znRecord.getListFields();
    for (String partition : currentMapping.keySet()) {
        // make sure these are all MASTER
        List<String> preferenceList = preferenceLists.get(partition);
        Assert.assertNotNull(preferenceList, "invalid preference list for " + partition);
        Assert.assertEquals(preferenceList.size(), 1, "invalid preference list for " + partition);
    }
    // now assign a replica to the first node in the current mapping, and add a second node
    allNodes.add(NODES[1]);
    liveNodes.add(NODES[1]);
    stateCount = STATE_MODEL.getStateCountMap(liveNodes.size(), REPLICA_COUNT);
    for (String partition : PARTITIONS) {
        currentMapping.get(partition).put(NODES[0], "MASTER");
    }
    znRecord = new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount).computePartitionAssignment(allNodes, liveNodes, currentMapping, null);
    preferenceLists = znRecord.getListFields();
    for (String partition : currentMapping.keySet()) {
        List<String> preferenceList = preferenceLists.get(partition);
        Assert.assertNotNull(preferenceList, "invalid preference list for " + partition);
        Assert.assertEquals(preferenceList.size(), 2, "invalid preference list for " + partition);
        Assert.assertEquals(preferenceList.get(0), NODES[0], "invalid preference list for " + partition);
        Assert.assertEquals(preferenceList.get(1), NODES[1], "invalid preference list for " + partition);
    }
    // now set the current mapping to reflect this update and make sure that it distributes masters
    for (String partition : PARTITIONS) {
        currentMapping.get(partition).put(NODES[1], "SLAVE");
    }
    znRecord = new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount).computePartitionAssignment(allNodes, liveNodes, currentMapping, null);
    preferenceLists = znRecord.getListFields();
    Set<String> firstNodes = Sets.newHashSet();
    for (String partition : currentMapping.keySet()) {
        List<String> preferenceList = preferenceLists.get(partition);
        Assert.assertNotNull(preferenceList, "invalid preference list for " + partition);
        Assert.assertEquals(preferenceList.size(), 2, "invalid preference list for " + partition);
        firstNodes.add(preferenceList.get(0));
    }
    Assert.assertEquals(firstNodes.size(), 2, "masters not evenly distributed");
    // set a mapping corresponding to a valid mapping for 2 nodes, add a third node, check that the
    // new node is never the most preferred
    allNodes.add(NODES[2]);
    liveNodes.add(NODES[2]);
    stateCount = STATE_MODEL.getStateCountMap(liveNodes.size(), REPLICA_COUNT);
    // recall that the other two partitions are [MASTER, SLAVE], which is fine, just reorder one
    currentMapping.get(PARTITIONS[1]).put(NODES[0], "SLAVE");
    currentMapping.get(PARTITIONS[1]).put(NODES[1], "MASTER");
    znRecord = new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount).computePartitionAssignment(allNodes, liveNodes, currentMapping, null);
    preferenceLists = znRecord.getListFields();
    boolean newNodeUsed = false;
    for (String partition : currentMapping.keySet()) {
        List<String> preferenceList = preferenceLists.get(partition);
        Assert.assertNotNull(preferenceList, "invalid preference list for " + partition);
        Assert.assertEquals(preferenceList.size(), 2, "invalid preference list for " + partition);
        if (preferenceList.contains(NODES[2])) {
            newNodeUsed = true;
            Assert.assertEquals(preferenceList.get(1), NODES[2], "newly added node not at preference list tail for " + partition);
        }
    }
    Assert.assertTrue(newNodeUsed, "not using " + NODES[2]);
    // evenly across all nodes
    for (String partition : PARTITIONS) {
        currentMapping.get(partition).clear();
    }
    currentMapping.get(PARTITIONS[0]).put(NODES[0], "MASTER");
    currentMapping.get(PARTITIONS[0]).put(NODES[1], "SLAVE");
    currentMapping.get(PARTITIONS[1]).put(NODES[1], "MASTER");
    currentMapping.get(PARTITIONS[1]).put(NODES[2], "SLAVE");
    currentMapping.get(PARTITIONS[2]).put(NODES[0], "MASTER");
    currentMapping.get(PARTITIONS[2]).put(NODES[2], "SLAVE");
    znRecord = new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount).computePartitionAssignment(allNodes, liveNodes, currentMapping, null);
    preferenceLists = znRecord.getListFields();
    firstNodes.clear();
    Set<String> secondNodes = Sets.newHashSet();
    for (String partition : currentMapping.keySet()) {
        List<String> preferenceList = preferenceLists.get(partition);
        Assert.assertNotNull(preferenceList, "invalid preference list for " + partition);
        Assert.assertEquals(preferenceList.size(), 2, "invalid preference list for " + partition);
        firstNodes.add(preferenceList.get(0));
        secondNodes.add(preferenceList.get(1));
    }
    Assert.assertEquals(firstNodes.size(), 3, "masters not distributed evenly");
    Assert.assertEquals(secondNodes.size(), 3, "slaves not distributed evenly");
    // remove a node now, but use the current mapping with everything balanced just prior
    liveNodes.remove(0);
    stateCount = STATE_MODEL.getStateCountMap(liveNodes.size(), REPLICA_COUNT);
    // remove all references of n0 from the mapping, keep everything else in a legal state
    for (String partition : PARTITIONS) {
        currentMapping.get(partition).clear();
    }
    currentMapping.get(PARTITIONS[0]).put(NODES[1], "MASTER");
    currentMapping.get(PARTITIONS[1]).put(NODES[1], "MASTER");
    currentMapping.get(PARTITIONS[1]).put(NODES[2], "SLAVE");
    currentMapping.get(PARTITIONS[2]).put(NODES[2], "MASTER");
    znRecord = new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount).computePartitionAssignment(allNodes, liveNodes, currentMapping, null);
    preferenceLists = znRecord.getListFields();
    for (String partition : currentMapping.keySet()) {
        List<String> preferenceList = preferenceLists.get(partition);
        Assert.assertNotNull(preferenceList, "invalid preference list for " + partition);
        Assert.assertEquals(preferenceList.size(), 2, "invalid preference list for " + partition);
        Map<String, String> stateMap = currentMapping.get(partition);
        for (String participant : stateMap.keySet()) {
            Assert.assertTrue(preferenceList.contains(participant), "minimal movement violated for " + partition);
        }
        for (String participant : preferenceList) {
            if (!stateMap.containsKey(participant)) {
                Assert.assertNotSame(preferenceList.get(0), participant, "newly moved replica should not be master for " + partition);
            }
        }
    }
    // finally, adjust the current mapping to reflect 2 nodes and make sure everything's even again
    for (String partition : PARTITIONS) {
        currentMapping.get(partition).clear();
    }
    currentMapping.get(PARTITIONS[0]).put(NODES[1], "MASTER");
    currentMapping.get(PARTITIONS[0]).put(NODES[2], "SLAVE");
    currentMapping.get(PARTITIONS[1]).put(NODES[1], "SLAVE");
    currentMapping.get(PARTITIONS[1]).put(NODES[2], "MASTER");
    currentMapping.get(PARTITIONS[2]).put(NODES[1], "SLAVE");
    currentMapping.get(PARTITIONS[2]).put(NODES[2], "MASTER");
    znRecord = new AutoRebalanceStrategy(RESOURCE_NAME, partitions, stateCount).computePartitionAssignment(allNodes, liveNodes, currentMapping, null);
    preferenceLists = znRecord.getListFields();
    firstNodes.clear();
    for (String partition : currentMapping.keySet()) {
        List<String> preferenceList = preferenceLists.get(partition);
        Assert.assertNotNull(preferenceList, "invalid preference list for " + partition);
        Assert.assertEquals(preferenceList.size(), 2, "invalid preference list for " + partition);
        firstNodes.add(preferenceList.get(0));
    }
    Assert.assertEquals(firstNodes.size(), 2, "masters not evenly distributed");
}

Example 3 with AutoRebalanceStrategy

use of org.apache.helix.controller.rebalancer.strategy.AutoRebalanceStrategy in project helix by apache.

the class TestAutoRebalanceStrategy method test.

@Test
public void test() {
    int nPartitions = 16;
    final String resourceName = "something";
    final List<String> instanceNames = // Initialize to 4 unique strings
    Arrays.asList("node-1", "node-2", "node-3", "node-4");
    final int nReplicas = 3;
    List<String> partitions = new ArrayList<String>(nPartitions);
    for (int i = 0; i < nPartitions; i++) {
        partitions.add(Integer.toString(i));
    }
    LinkedHashMap<String, Integer> states = new LinkedHashMap<String, Integer>(2);
    states.put("OFFLINE", 0);
    states.put("ONLINE", nReplicas);
    AutoRebalanceStrategy strategy = new AutoRebalanceStrategy(resourceName, partitions, states);
    ZNRecord znRecord = strategy.computePartitionAssignment(instanceNames, instanceNames, new HashMap<String, Map<String, String>>(0), null);
    for (List p : znRecord.getListFields().values()) {
        Assert.assertEquals(p.size(), nReplicas);
    }
}

Example 4 with AutoRebalanceStrategy

use of org.apache.helix.controller.rebalancer.strategy.AutoRebalanceStrategy in project helix by apache.

the class TestAutoRebalanceStrategyImbalanceAssignment method testAssignment.

private void testAssignment(int nPartitions, int nReplicas, int nNode) {
    final List<String> instanceNames = new ArrayList<>();
    for (int i = 0; i < nNode; i++) {
        instanceNames.add("localhost_" + i);
    }
    List<String> partitions = new ArrayList<>(nPartitions);
    for (int i = 0; i < nPartitions; i++) {
        partitions.add(Integer.toString(i));
    }
    LinkedHashMap<String, Integer> states = new LinkedHashMap<>(2);
    states.put("OFFLINE", 0);
    states.put("ONLINE", nReplicas);
    AutoRebalanceStrategy strategy = new AutoRebalanceStrategy(resourceName, partitions, states);
    ZNRecord record = strategy.computePartitionAssignment(instanceNames, instanceNames, new HashMap<String, Map<String, String>>(0), new ClusterDataCache());
    for (Map<String, String> stateMapping : record.getMapFields().values()) {
        Assert.assertEquals(stateMapping.size(), nReplicas);
    }
}

Example 5 with AutoRebalanceStrategy

use of org.apache.helix.controller.rebalancer.strategy.AutoRebalanceStrategy in project helix by apache.

the class AbstractRebalancer method getRebalanceStrategy.

protected RebalanceStrategy getRebalanceStrategy(String rebalanceStrategyName, List<String> partitions, String resourceName, LinkedHashMap<String, Integer> stateCountMap, int maxPartition) {
    RebalanceStrategy rebalanceStrategy;
    if (rebalanceStrategyName == null || rebalanceStrategyName.equalsIgnoreCase(RebalanceStrategy.DEFAULT_REBALANCE_STRATEGY)) {
        rebalanceStrategy = new AutoRebalanceStrategy(resourceName, partitions, stateCountMap, maxPartition);
    } else {
        try {
            rebalanceStrategy = RebalanceStrategy.class.cast(HelixUtil.loadClass(getClass(), rebalanceStrategyName).newInstance());
            rebalanceStrategy.init(resourceName, partitions, stateCountMap, maxPartition);
        } catch (ClassNotFoundException ex) {
            throw new HelixException("Exception while invoking custom rebalance strategy class: " + rebalanceStrategyName, ex);
        } catch (InstantiationException ex) {
            throw new HelixException("Exception while invoking custom rebalance strategy class: " + rebalanceStrategyName, ex);
        } catch (IllegalAccessException ex) {
            throw new HelixException("Exception while invoking custom rebalance strategy class: " + rebalanceStrategyName, ex);
        }
    }
    return rebalanceStrategy;
}