Examples with Node org.apache.helix.controller.rebalancer.topology.Node used on opensource projects

Example 1 with Node

use of org.apache.helix.controller.rebalancer.topology.Node in project helix by apache.

the class MultiRoundCrushRebalanceStrategy method generateZNRecord.

private ZNRecord generateZNRecord(String resource, List<String> partitions, Map<String, Map<String, List<Node>>> partitionStateMapping) {
    Map<String, List<String>> newPreferences = new HashMap<String, List<String>>();
    for (int i = 0; i < partitions.size(); i++) {
        String partitionName = partitions.get(i);
        Map<String, List<Node>> stateNodeMap = partitionStateMapping.get(partitionName);
        for (String state : _stateCountMap.keySet()) {
            List<Node> nodes = stateNodeMap.get(state);
            List<String> nodeList = new ArrayList<String>();
            for (int j = 0; j < nodes.size(); j++) {
                nodeList.add(nodes.get(j).getName());
            }
            if (!newPreferences.containsKey(partitionName)) {
                newPreferences.put(partitionName, new ArrayList<String>());
            }
            newPreferences.get(partitionName).addAll(nodeList);
        }
    }
    ZNRecord result = new ZNRecord(resource);
    result.setListFields(newPreferences);
    return result;
}

Example 2 with Node

use of org.apache.helix.controller.rebalancer.topology.Node in project helix by apache.

the class MultiRoundCrushRebalanceStrategy method singleZoneMapping.

/**
 * Compute mapping of partition to node in a single zone.
 * Assumption: A partition should have only one replica at one zone.
 * Will apply CRUSH multiple times until all partitions are mostly even distributed.
 *
 * @param zone       the zone
 * @param partitions partitions to be assigned to nodes in the given zone.
 * @return partition to node mapping in this zone.
 */
private Map<String, Node> singleZoneMapping(Node zone, List<String> partitions) {
    if (zone.isFailed() || zone.getWeight() == 0 || partitions.isEmpty()) {
        return Collections.emptyMap();
    }
    long totalWeight = zone.getWeight();
    int totalPartition = partitions.size();
    // node to all its assigned partitions.
    Map<Node, List<String>> nodePartitionsMap = new HashMap<>();
    Map<Node, List<String>> prevNodePartitionsMap = new HashMap<>();
    List<String> partitionsToAssign = new ArrayList<>(partitions);
    Map<Node, List<String>> toRemovedMap = new HashMap<>();
    int iteration = 0;
    Node root = zone;
    boolean noAssignmentFound = false;
    while (iteration++ < MAX_ITERNATION && !noAssignmentFound) {
        copyAssignment(nodePartitionsMap, prevNodePartitionsMap);
        for (Map.Entry<Node, List<String>> e : toRemovedMap.entrySet()) {
            List<String> curAssignedPartitions = nodePartitionsMap.get(e.getKey());
            List<String> toRemoved = e.getValue();
            curAssignedPartitions.removeAll(toRemoved);
            partitionsToAssign.addAll(toRemoved);
        }
        for (String p : partitionsToAssign) {
            long pData = p.hashCode();
            List<Node> nodes;
            try {
                nodes = select(root, _clusterTopo.getEndNodeType(), pData, 1);
            } catch (IllegalStateException e) {
                nodePartitionsMap = prevNodePartitionsMap;
                noAssignmentFound = true;
                break;
            }
            for (Node n : nodes) {
                if (!nodePartitionsMap.containsKey(n)) {
                    nodePartitionsMap.put(n, new ArrayList<String>());
                }
                nodePartitionsMap.get(n).add(p);
            }
            root = recalculateWeight(zone, totalWeight, totalPartition, nodePartitionsMap, partitions, toRemovedMap);
        }
        partitionsToAssign.clear();
    }
    Map<String, Node> partitionMap = new HashMap<>();
    for (Map.Entry<Node, List<String>> e : nodePartitionsMap.entrySet()) {
        Node n = e.getKey();
        List<String> assigned = e.getValue();
        for (String p : assigned) {
            partitionMap.put(p, n);
        }
    }
    return partitionMap;
}

Example 3 with Node

use of org.apache.helix.controller.rebalancer.topology.Node in project helix by apache.

the class TestTopology method testCreateClusterTopologyWithDefaultTopology.

@Test
public void testCreateClusterTopologyWithDefaultTopology() {
    ClusterConfig clusterConfig = new ClusterConfig("Test_Cluster");
    clusterConfig.setTopologyAwareEnabled(true);
    List<String> allNodes = new ArrayList<String>();
    List<String> liveNodes = new ArrayList<String>();
    Map<String, InstanceConfig> instanceConfigMap = new HashMap<String, InstanceConfig>();
    Map<String, Integer> nodeToWeightMap = new HashMap<String, Integer>();
    for (int i = 0; i < 100; i++) {
        String instance = "localhost_" + i;
        InstanceConfig config = new InstanceConfig(instance);
        String zoneId = "rack_" + i / 10;
        config.setZoneId(zoneId);
        config.setHostName(instance);
        config.setPort("9000");
        allNodes.add(instance);
        int weight = 0;
        if (i % 10 != 0) {
            liveNodes.add(instance);
            weight = 1000;
            if (i % 3 == 0) {
                // set random instance weight.
                weight = (i + 1) * 100;
                config.setWeight(weight);
            }
        }
        instanceConfigMap.put(instance, config);
        if (!nodeToWeightMap.containsKey(zoneId)) {
            nodeToWeightMap.put(zoneId, 0);
        }
        nodeToWeightMap.put(zoneId, nodeToWeightMap.get(zoneId) + weight);
    }
    Topology topo = new Topology(allNodes, liveNodes, instanceConfigMap, clusterConfig);
    Assert.assertTrue(topo.getEndNodeType().equals(Topology.Types.INSTANCE.name()));
    Assert.assertTrue(topo.getFaultZoneType().equals(Topology.Types.ZONE.name()));
    List<Node> faultZones = topo.getFaultZones();
    Assert.assertEquals(faultZones.size(), 10);
    Node root = topo.getRootNode();
    Assert.assertEquals(root.getChildrenCount(Topology.Types.ZONE.name()), 10);
    Assert.assertEquals(root.getChildrenCount(topo.getEndNodeType()), 100);
    // validate weights.
    for (Node rack : root.getChildren()) {
        Assert.assertEquals(rack.getWeight(), (long) nodeToWeightMap.get(rack.getName()));
    }
}

Example 4 with Node

use of org.apache.helix.controller.rebalancer.topology.Node in project helix by apache.

the class TestTopology method testCreateClusterTopology.

@Test
public void testCreateClusterTopology() {
    ClusterConfig clusterConfig = new ClusterConfig("Test_Cluster");
    String topology = "/Rack/Sub-Rack/Host/Instance";
    clusterConfig.setTopology(topology);
    clusterConfig.setFaultZoneType("Sub-Rack");
    clusterConfig.setTopologyAwareEnabled(true);
    List<String> allNodes = new ArrayList<String>();
    List<String> liveNodes = new ArrayList<String>();
    Map<String, InstanceConfig> instanceConfigMap = new HashMap<String, InstanceConfig>();
    Map<String, Integer> nodeToWeightMap = new HashMap<String, Integer>();
    for (int i = 0; i < 100; i++) {
        String instance = "localhost_" + i;
        InstanceConfig config = new InstanceConfig(instance);
        String rack_id = "rack_" + i / 25;
        String sub_rack_id = "subrack-" + i / 5;
        String domain = String.format("Rack=%s, Sub-Rack=%s, Host=%s", rack_id, sub_rack_id, instance);
        config.setDomain(domain);
        config.setHostName(instance);
        config.setPort("9000");
        allNodes.add(instance);
        int weight = 0;
        if (i % 10 != 0) {
            liveNodes.add(instance);
            weight = 1000;
            if (i % 3 == 0) {
                // set random instance weight.
                weight = (i + 1) * 100;
                config.setWeight(weight);
            }
        }
        instanceConfigMap.put(instance, config);
        if (!nodeToWeightMap.containsKey(rack_id)) {
            nodeToWeightMap.put(rack_id, 0);
        }
        nodeToWeightMap.put(rack_id, nodeToWeightMap.get(rack_id) + weight);
        if (!nodeToWeightMap.containsKey(sub_rack_id)) {
            nodeToWeightMap.put(sub_rack_id, 0);
        }
        nodeToWeightMap.put(sub_rack_id, nodeToWeightMap.get(sub_rack_id) + weight);
    }
    Topology topo = new Topology(allNodes, liveNodes, instanceConfigMap, clusterConfig);
    Assert.assertTrue(topo.getEndNodeType().equals("Instance"));
    Assert.assertTrue(topo.getFaultZoneType().equals("Sub-Rack"));
    List<Node> faultZones = topo.getFaultZones();
    Assert.assertEquals(faultZones.size(), 20);
    Node root = topo.getRootNode();
    Assert.assertEquals(root.getChildrenCount("Rack"), 4);
    Assert.assertEquals(root.getChildrenCount("Sub-Rack"), 20);
    Assert.assertEquals(root.getChildrenCount("Host"), 100);
    Assert.assertEquals(root.getChildrenCount("Instance"), 100);
    // validate weights.
    for (Node rack : root.getChildren()) {
        Assert.assertEquals(rack.getWeight(), (long) nodeToWeightMap.get(rack.getName()));
        for (Node subRack : rack.getChildren()) {
            Assert.assertEquals(subRack.getWeight(), (long) nodeToWeightMap.get(subRack.getName()));
        }
    }
}

Example 5 with Node

use of org.apache.helix.controller.rebalancer.topology.Node in project helix by apache.

the class CrushRebalanceStrategy method doSelect.

private void doSelect(Node topNode, long input, int rf, List<Node> selectedNodes, Set<Node> selectedZones) {
    String zoneType = _clusterTopo.getFaultZoneType();
    String endNodeType = _clusterTopo.getEndNodeType();
    if (!zoneType.equals(endNodeType)) {
        // pick fault zones first
        List<Node> zones = placementAlgorithm.select(topNode, input, rf, zoneType, nodeAlreadySelected(selectedZones));
        // add the racks to the selected racks
        selectedZones.addAll(zones);
        // pick one end node from each fault zone.
        for (Node zone : zones) {
            List<Node> endNode = placementAlgorithm.select(zone, input, 1, endNodeType);
            selectedNodes.addAll(endNode);
        }
    } else {
        // pick end node directly
        List<Node> nodes = placementAlgorithm.select(topNode, input, rf, endNodeType, nodeAlreadySelected(new HashSet(selectedNodes)));
        selectedNodes.addAll(nodes);
    }
}