Examples with NodeLabelsUpdateSchedulerEvent org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent used on opensource projects

Example 1 with NodeLabelsUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent in project hadoop by apache.

the class TestCapacitySchedulerNodeLabelUpdate method testComplexResourceUsageWhenNodeUpdatesPartition.

@Test(timeout = 60000)
public void testComplexResourceUsageWhenNodeUpdatesPartition() throws Exception {
    /*
     * This test is similar to testResourceUsageWhenNodeUpdatesPartition, this
     * will include multiple applications, multiple users and multiple
     * containers running on a single node, size of each container is 1G
     *
     * Node 1
     * ------
     * App1-container3
     * App2-container2
     * App2-Container3
     *
     * Node 2
     * ------
     * App2-container1
     * App1-container1
     * App1-container2
     */
    // set node -> label
    mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z"));
    // set mapping:
    // h1 -> x
    // h2 -> y
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
    // inject node label manager
    MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 80000);
    MockNM nm2 = rm.registerNode("h2:1234", 80000);
    // app1
    RMApp app1 = rm.submitApp(GB, "app", "u1", null, "a");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2);
    // c2 on n1, c3 on n2
    am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
    ContainerId containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    Assert.assertTrue(rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED));
    am1.allocate("*", GB, 1, new ArrayList<ContainerId>());
    containerId = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
    Assert.assertTrue(rm.waitForState(nm2, containerId, RMContainerState.ALLOCATED));
    // app2
    RMApp app2 = rm.submitApp(GB, "app", "u2", null, "a");
    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2);
    // c2/c3 on n1
    am2.allocate("*", GB, 2, new ArrayList<ContainerId>(), "x");
    containerId = ContainerId.newContainerId(am2.getApplicationAttemptId(), 3);
    Assert.assertTrue(rm.waitForState(nm1, containerId, RMContainerState.ALLOCATED));
    // check used resource:
    // queue-a used x=1G, ""=1G
    checkUsedResource(rm, "a", 3 * GB, "x");
    checkUsedResource(rm, "a", 3 * GB);
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    FiCaSchedulerApp application1 = cs.getApplicationAttempt(am1.getApplicationAttemptId());
    FiCaSchedulerApp application2 = cs.getApplicationAttempt(am2.getApplicationAttemptId());
    // change h1's label to z
    cs.handle(new NodeLabelsUpdateSchedulerEvent(ImmutableMap.of(nm1.getNodeId(), toSet("z"))));
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 3 * GB, "z");
    checkUsedResource(rm, "a", 3 * GB);
    checkUsedResource(rm, "root", 0, "x");
    checkUsedResource(rm, "root", 3 * GB, "z");
    checkUsedResource(rm, "root", 3 * GB);
    checkUserUsedResource(rm, "a", "u1", "x", 0 * GB);
    checkUserUsedResource(rm, "a", "u1", "z", 1 * GB);
    checkUserUsedResource(rm, "a", "u1", "", 2 * GB);
    checkUserUsedResource(rm, "a", "u2", "x", 0 * GB);
    checkUserUsedResource(rm, "a", "u2", "z", 2 * GB);
    checkUserUsedResource(rm, "a", "u2", "", 1 * GB);
    Assert.assertEquals(0, application1.getAppAttemptResourceUsage().getUsed("x").getMemorySize());
    Assert.assertEquals(1 * GB, application1.getAppAttemptResourceUsage().getUsed("z").getMemorySize());
    Assert.assertEquals(2 * GB, application1.getAppAttemptResourceUsage().getUsed("").getMemorySize());
    Assert.assertEquals(0, application2.getAppAttemptResourceUsage().getUsed("x").getMemorySize());
    Assert.assertEquals(2 * GB, application2.getAppAttemptResourceUsage().getUsed("z").getMemorySize());
    Assert.assertEquals(1 * GB, application2.getAppAttemptResourceUsage().getUsed("").getMemorySize());
    rm.close();
}

Example 2 with NodeLabelsUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent in project hadoop by apache.

the class TestCapacitySchedulerNodeLabelUpdate method testResourceUsageWhenNodeUpdatesPartition.

@Test
public void testResourceUsageWhenNodeUpdatesPartition() throws Exception {
    // set node -> label
    mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z"));
    // set mapping:
    // h1 -> x
    // h2 -> y
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
    // inject node label manager
    MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 8000);
    MockNM nm2 = rm.registerNode("h2:1234", 8000);
    MockNM nm3 = rm.registerNode("h3:1234", 8000);
    ContainerId containerId1;
    ContainerId containerId2;
    // launch an app to queue a1 (label = x), and check all container will
    // be allocated in h1
    RMApp app1 = rm.submitApp(GB, "app", "user", null, "a");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm3);
    // request a container.
    am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
    containerId1 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
    containerId2 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    Assert.assertTrue(rm.waitForState(nm1, containerId2, RMContainerState.ALLOCATED));
    // check used resource:
    // queue-a used x=1G, ""=1G
    checkUsedResource(rm, "a", 1024, "x");
    checkUsedResource(rm, "a", 1024);
    checkUsedCapacity(rm, "a", 1024, 8000, "x");
    checkUsedCapacity(rm, "a", 1024, 8000);
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    FiCaSchedulerApp app = cs.getApplicationAttempt(am1.getApplicationAttemptId());
    // change h1's label to z
    mgr.replaceLabelsOnNode(ImmutableMap.of(nm1.getNodeId(), toSet("z")));
    cs.handle(new NodeLabelsUpdateSchedulerEvent(ImmutableMap.of(nm1.getNodeId(), toSet("z"))));
    Thread.sleep(100);
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 1024, "z");
    checkUsedResource(rm, "a", 1024);
    checkUsedCapacity(rm, "a", 0, 8000, "x");
    checkUsedCapacity(rm, "a", 1024, 8000, "z");
    checkUsedCapacity(rm, "a", 1024, 8000);
    checkUsedResource(rm, "root", 0, "x");
    checkUsedResource(rm, "root", 1024, "z");
    checkUsedResource(rm, "root", 1024);
    checkUserUsedResource(rm, "a", "user", "x", 0);
    checkUserUsedResource(rm, "a", "user", "z", 1024);
    Assert.assertEquals(0, app.getAppAttemptResourceUsage().getUsed("x").getMemorySize());
    Assert.assertEquals(1024, app.getAppAttemptResourceUsage().getUsed("z").getMemorySize());
    // change h1's label to y
    mgr.replaceLabelsOnNode(ImmutableMap.of(nm1.getNodeId(), toSet("y")));
    cs.handle(new NodeLabelsUpdateSchedulerEvent(ImmutableMap.of(nm1.getNodeId(), toSet("y"))));
    Thread.sleep(100);
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 1024, "y");
    checkUsedResource(rm, "a", 0, "z");
    checkUsedResource(rm, "a", 1024);
    checkUsedCapacity(rm, "a", 0, 8000, "x");
    checkUsedCapacity(rm, "a", 1024, 16000, "y");
    checkUsedCapacity(rm, "a", 0, 8000, "z");
    checkUsedCapacity(rm, "a", 1024, 8000);
    checkUsedResource(rm, "root", 0, "x");
    checkUsedResource(rm, "root", 1024, "y");
    checkUsedResource(rm, "root", 0, "z");
    checkUsedResource(rm, "root", 1024);
    checkUserUsedResource(rm, "a", "user", "x", 0);
    checkUserUsedResource(rm, "a", "user", "y", 1024);
    checkUserUsedResource(rm, "a", "user", "z", 0);
    Assert.assertEquals(0, app.getAppAttemptResourceUsage().getUsed("x").getMemorySize());
    Assert.assertEquals(1024, app.getAppAttemptResourceUsage().getUsed("y").getMemorySize());
    Assert.assertEquals(0, app.getAppAttemptResourceUsage().getUsed("z").getMemorySize());
    // change h1's label to no label
    Set<String> emptyLabels = new HashSet<>();
    Map<NodeId, Set<String>> map = ImmutableMap.of(nm1.getNodeId(), emptyLabels);
    mgr.replaceLabelsOnNode(map);
    cs.handle(new NodeLabelsUpdateSchedulerEvent(map));
    Thread.sleep(100);
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 0, "y");
    checkUsedResource(rm, "a", 0, "z");
    checkUsedResource(rm, "a", 2048);
    checkUsedCapacity(rm, "a", 0, 8000, "x");
    checkUsedCapacity(rm, "a", 0, 8000, "y");
    checkUsedCapacity(rm, "a", 0, 8000, "z");
    checkUsedCapacity(rm, "a", 2048, 16000);
    checkUsedResource(rm, "root", 0, "x");
    checkUsedResource(rm, "root", 0, "y");
    checkUsedResource(rm, "root", 0, "z");
    checkUsedResource(rm, "root", 2048);
    checkUserUsedResource(rm, "a", "user", "x", 0);
    checkUserUsedResource(rm, "a", "user", "y", 0);
    checkUserUsedResource(rm, "a", "user", "z", 0);
    checkUserUsedResource(rm, "a", "user", "", 2048);
    Assert.assertEquals(0, app.getAppAttemptResourceUsage().getUsed("x").getMemorySize());
    Assert.assertEquals(0, app.getAppAttemptResourceUsage().getUsed("y").getMemorySize());
    Assert.assertEquals(0, app.getAppAttemptResourceUsage().getUsed("z").getMemorySize());
    Assert.assertEquals(2048, app.getAppAttemptResourceUsage().getUsed("").getMemorySize());
    // Finish the two containers, we should see used resource becomes 0
    cs.completedContainer(cs.getRMContainer(containerId2), ContainerStatus.newInstance(containerId2, ContainerState.COMPLETE, "", ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), RMContainerEventType.KILL);
    cs.completedContainer(cs.getRMContainer(containerId1), ContainerStatus.newInstance(containerId1, ContainerState.COMPLETE, "", ContainerExitStatus.KILLED_BY_RESOURCEMANAGER), RMContainerEventType.KILL);
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 0, "y");
    checkUsedResource(rm, "a", 0, "z");
    checkUsedResource(rm, "a", 0);
    checkUsedCapacity(rm, "a", 0, 8000, "x");
    checkUsedCapacity(rm, "a", 0, 8000, "y");
    checkUsedCapacity(rm, "a", 0, 8000, "z");
    checkUsedCapacity(rm, "a", 0, 16000);
    checkUsedResource(rm, "root", 0, "x");
    checkUsedResource(rm, "root", 0, "y");
    checkUsedResource(rm, "root", 0, "z");
    checkUsedResource(rm, "root", 0);
    checkUserUsedResource(rm, "a", "user", "x", 0);
    checkUserUsedResource(rm, "a", "user", "y", 0);
    checkUserUsedResource(rm, "a", "user", "z", 0);
    checkUserUsedResource(rm, "a", "user", "", 0);
    rm.close();
}

Example 3 with NodeLabelsUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent in project hadoop by apache.

the class TestCapacitySchedulerNodeLabelUpdate method testAMResourceUsageWhenNodeUpdatesPartition.

@Test(timeout = 60000)
public void testAMResourceUsageWhenNodeUpdatesPartition() throws Exception {
    // set node -> label
    mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z"));
    // set mapping:
    // h1 -> x
    // h2 -> y
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
    // inject node label manager
    MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 8000);
    rm.registerNode("h2:1234", 8000);
    rm.registerNode("h3:1234", 8000);
    ContainerId containerId2;
    // launch an app to queue a1 (label = x), and check all container will
    // be allocated in h1
    RMApp app1 = rm.submitApp(GB, "app", "user", null, "a", "x");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
    // request a container.
    am1.allocate("*", GB, 1, new ArrayList<ContainerId>(), "x");
    ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
    containerId2 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    Assert.assertTrue(rm.waitForState(nm1, containerId2, RMContainerState.ALLOCATED));
    // check used resource:
    // queue-a used x=2G
    checkUsedResource(rm, "a", 2048, "x");
    checkAMUsedResource(rm, "a", 1024, "x");
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    FiCaSchedulerApp app = cs.getApplicationAttempt(am1.getApplicationAttemptId());
    // change h1's label to z
    cs.handle(new NodeLabelsUpdateSchedulerEvent(ImmutableMap.of(nm1.getNodeId(), toSet("z"))));
    // Now the resources also should change from x to z. Verify AM and normal
    // used resource are successfully changed.
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 2048, "z");
    checkAMUsedResource(rm, "a", 0, "x");
    checkAMUsedResource(rm, "a", 1024, "z");
    checkUserUsedResource(rm, "a", "user", "x", 0);
    checkUserUsedResource(rm, "a", "user", "z", 2048);
    Assert.assertEquals(0, app.getAppAttemptResourceUsage().getAMUsed("x").getMemorySize());
    Assert.assertEquals(1024, app.getAppAttemptResourceUsage().getAMUsed("z").getMemorySize());
    // change h1's label to no label
    Set<String> emptyLabels = new HashSet<>();
    Map<NodeId, Set<String>> map = ImmutableMap.of(nm1.getNodeId(), emptyLabels);
    cs.handle(new NodeLabelsUpdateSchedulerEvent(map));
    checkUsedResource(rm, "a", 0, "x");
    checkUsedResource(rm, "a", 0, "z");
    checkUsedResource(rm, "a", 2048);
    checkAMUsedResource(rm, "a", 0, "x");
    checkAMUsedResource(rm, "a", 0, "z");
    checkAMUsedResource(rm, "a", 1024);
    checkUserUsedResource(rm, "a", "user", "x", 0);
    checkUserUsedResource(rm, "a", "user", "z", 0);
    checkUserUsedResource(rm, "a", "user", "", 2048);
    Assert.assertEquals(0, app.getAppAttemptResourceUsage().getAMUsed("x").getMemorySize());
    Assert.assertEquals(0, app.getAppAttemptResourceUsage().getAMUsed("z").getMemorySize());
    Assert.assertEquals(1024, app.getAppAttemptResourceUsage().getAMUsed("").getMemorySize());
    rm.close();
}

Example 4 with NodeLabelsUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsUpdateSchedulerEvent in project hadoop by apache.

the class RMNodeLabelsManager method updateResourceMappings.

@SuppressWarnings("unchecked")
private void updateResourceMappings(Map<String, Host> before, Map<String, Host> after) {
    // Get NMs in before only
    Set<NodeId> allNMs = new HashSet<NodeId>();
    for (Entry<String, Host> entry : before.entrySet()) {
        allNMs.addAll(entry.getValue().nms.keySet());
    }
    for (Entry<String, Host> entry : after.entrySet()) {
        allNMs.addAll(entry.getValue().nms.keySet());
    }
    // Map used to notify RM
    Map<NodeId, Set<String>> newNodeToLabelsMap = new HashMap<NodeId, Set<String>>();
    // traverse all nms
    for (NodeId nodeId : allNMs) {
        Node oldNM;
        if ((oldNM = getNMInNodeSet(nodeId, before, true)) != null) {
            Set<String> oldLabels = getLabelsByNode(nodeId, before);
            // no label in the past
            if (oldLabels.isEmpty()) {
                // update labels
                RMNodeLabel label = labelCollections.get(NO_LABEL);
                label.removeNode(oldNM.resource);
                // update queues, all queue can access this node
                for (Queue q : queueCollections.values()) {
                    Resources.subtractFrom(q.resource, oldNM.resource);
                }
            } else {
                // update labels
                for (String labelName : oldLabels) {
                    RMNodeLabel label = labelCollections.get(labelName);
                    if (null == label) {
                        continue;
                    }
                    label.removeNode(oldNM.resource);
                }
                // update queues, only queue can access this node will be subtract
                for (Queue q : queueCollections.values()) {
                    if (isNodeUsableByQueue(oldLabels, q)) {
                        Resources.subtractFrom(q.resource, oldNM.resource);
                    }
                }
            }
        }
        Node newNM;
        if ((newNM = getNMInNodeSet(nodeId, after, true)) != null) {
            Set<String> newLabels = getLabelsByNode(nodeId, after);
            newNodeToLabelsMap.put(nodeId, ImmutableSet.copyOf(newLabels));
            // no label in the past
            if (newLabels.isEmpty()) {
                // update labels
                RMNodeLabel label = labelCollections.get(NO_LABEL);
                label.addNode(newNM.resource);
                // update queues, all queue can access this node
                for (Queue q : queueCollections.values()) {
                    Resources.addTo(q.resource, newNM.resource);
                }
            } else {
                // update labels
                for (String labelName : newLabels) {
                    RMNodeLabel label = labelCollections.get(labelName);
                    label.addNode(newNM.resource);
                }
                // update queues, only queue can access this node will be subtract
                for (Queue q : queueCollections.values()) {
                    if (isNodeUsableByQueue(newLabels, q)) {
                        Resources.addTo(q.resource, newNM.resource);
                    }
                }
            }
        }
    }
    // Notify RM
    if (rmContext != null && rmContext.getDispatcher() != null) {
        rmContext.getDispatcher().getEventHandler().handle(new NodeLabelsUpdateSchedulerEvent(newNodeToLabelsMap));
    }
}