Examples with RMNodeLabel org.apache.hadoop.yarn.nodelabels.RMNodeLabel used on opensource projects

Example 1 with RMNodeLabel

use of org.apache.hadoop.yarn.nodelabels.RMNodeLabel in project hadoop by apache.

the class RMNodeLabelsManager method activateNode.

/*
   * Following methods are used for setting if a node is up and running, and it
   * will update running nodes resource
   */
public void activateNode(NodeId nodeId, Resource resource) {
    try {
        writeLock.lock();
        // save if we have a node before
        Map<String, Host> before = cloneNodeMap(ImmutableSet.of(nodeId));
        createHostIfNonExisted(nodeId.getHost());
        try {
            createNodeIfNonExisted(nodeId);
        } catch (IOException e) {
            LOG.error("This shouldn't happen, cannot get host in nodeCollection" + " associated to the node being activated");
            return;
        }
        Node nm = getNMInNodeSet(nodeId);
        nm.resource = resource;
        nm.running = true;
        // Add node in labelsCollection
        Set<String> labelsForNode = getLabelsByNode(nodeId);
        if (labelsForNode != null) {
            for (String label : labelsForNode) {
                RMNodeLabel labelInfo = labelCollections.get(label);
                if (labelInfo != null) {
                    labelInfo.addNodeId(nodeId);
                }
            }
        }
        // get the node after edition
        Map<String, Host> after = cloneNodeMap(ImmutableSet.of(nodeId));
        updateResourceMappings(before, after);
    } finally {
        writeLock.unlock();
    }
}

Example 2 with RMNodeLabel

use of org.apache.hadoop.yarn.nodelabels.RMNodeLabel in project hadoop by apache.

the class RMNodeLabelsManager method pullRMNodeLabelsInfo.

public List<RMNodeLabel> pullRMNodeLabelsInfo() {
    try {
        readLock.lock();
        List<RMNodeLabel> infos = new ArrayList<RMNodeLabel>();
        for (Entry<String, RMNodeLabel> entry : labelCollections.entrySet()) {
            RMNodeLabel label = entry.getValue();
            infos.add(label.getCopy());
        }
        Collections.sort(infos);
        return infos;
    } finally {
        readLock.unlock();
    }
}

Example 3 with RMNodeLabel

use of org.apache.hadoop.yarn.nodelabels.RMNodeLabel in project hadoop by apache.

the class TestRMNodeLabelsManager method checkNodeLabelInfo.

private void checkNodeLabelInfo(List<RMNodeLabel> infos, String labelName, int activeNMs, int memory) {
    for (RMNodeLabel info : infos) {
        if (info.getLabelName().equals(labelName)) {
            Assert.assertEquals(activeNMs, info.getNumActiveNMs());
            Assert.assertEquals(memory, info.getResource().getMemorySize());
            return;
        }
    }
    Assert.fail("Failed to find info has label=" + labelName);
}

Example 4 with RMNodeLabel

use of org.apache.hadoop.yarn.nodelabels.RMNodeLabel in project hadoop by apache.

the class RMNodeLabelsManager method getActiveNMCountPerLabel.

/*
   * Get active node count based on label.
   */
public int getActiveNMCountPerLabel(String label) {
    if (label == null) {
        return 0;
    }
    try {
        readLock.lock();
        RMNodeLabel labelInfo = labelCollections.get(label);
        return (labelInfo == null) ? 0 : labelInfo.getNumActiveNMs();
    } finally {
        readLock.unlock();
    }
}

Example 5 with RMNodeLabel

use of org.apache.hadoop.yarn.nodelabels.RMNodeLabel in project hadoop by apache.

the class RMNodeLabelsManager method updateResourceMappings.

@SuppressWarnings("unchecked")
private void updateResourceMappings(Map<String, Host> before, Map<String, Host> after) {
    // Get NMs in before only
    Set<NodeId> allNMs = new HashSet<NodeId>();
    for (Entry<String, Host> entry : before.entrySet()) {
        allNMs.addAll(entry.getValue().nms.keySet());
    }
    for (Entry<String, Host> entry : after.entrySet()) {
        allNMs.addAll(entry.getValue().nms.keySet());
    }
    // Map used to notify RM
    Map<NodeId, Set<String>> newNodeToLabelsMap = new HashMap<NodeId, Set<String>>();
    // traverse all nms
    for (NodeId nodeId : allNMs) {
        Node oldNM;
        if ((oldNM = getNMInNodeSet(nodeId, before, true)) != null) {
            Set<String> oldLabels = getLabelsByNode(nodeId, before);
            // no label in the past
            if (oldLabels.isEmpty()) {
                // update labels
                RMNodeLabel label = labelCollections.get(NO_LABEL);
                label.removeNode(oldNM.resource);
                // update queues, all queue can access this node
                for (Queue q : queueCollections.values()) {
                    Resources.subtractFrom(q.resource, oldNM.resource);
                }
            } else {
                // update labels
                for (String labelName : oldLabels) {
                    RMNodeLabel label = labelCollections.get(labelName);
                    if (null == label) {
                        continue;
                    }
                    label.removeNode(oldNM.resource);
                }
                // update queues, only queue can access this node will be subtract
                for (Queue q : queueCollections.values()) {
                    if (isNodeUsableByQueue(oldLabels, q)) {
                        Resources.subtractFrom(q.resource, oldNM.resource);
                    }
                }
            }
        }
        Node newNM;
        if ((newNM = getNMInNodeSet(nodeId, after, true)) != null) {
            Set<String> newLabels = getLabelsByNode(nodeId, after);
            newNodeToLabelsMap.put(nodeId, ImmutableSet.copyOf(newLabels));
            // no label in the past
            if (newLabels.isEmpty()) {
                // update labels
                RMNodeLabel label = labelCollections.get(NO_LABEL);
                label.addNode(newNM.resource);
                // update queues, all queue can access this node
                for (Queue q : queueCollections.values()) {
                    Resources.addTo(q.resource, newNM.resource);
                }
            } else {
                // update labels
                for (String labelName : newLabels) {
                    RMNodeLabel label = labelCollections.get(labelName);
                    label.addNode(newNM.resource);
                }
                // update queues, only queue can access this node will be subtract
                for (Queue q : queueCollections.values()) {
                    if (isNodeUsableByQueue(newLabels, q)) {
                        Resources.addTo(q.resource, newNM.resource);
                    }
                }
            }
        }
    }
    // Notify RM
    if (rmContext != null && rmContext.getDispatcher() != null) {
        rmContext.getDispatcher().getEventHandler().handle(new NodeLabelsUpdateSchedulerEvent(newNodeToLabelsMap));
    }
}