Examples with SchedulerNode org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode used on opensource projects

Example 11 with SchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode in project incubator-myriad by apache.

the class YarnNodeCapacityManager method afterSchedulerEventHandled.

@Override
public void afterSchedulerEventHandled(SchedulerEvent event) {
    switch(event.getType()) {
        case NODE_ADDED:
            if (!(event instanceof NodeAddedSchedulerEvent)) {
                LOGGER.error("{} not an instance of {}", event.getClass().getName(), NodeAddedSchedulerEvent.class.getName());
                return;
            }
            NodeAddedSchedulerEvent nodeAddedEvent = (NodeAddedSchedulerEvent) event;
            NodeId nodeId = nodeAddedEvent.getAddedRMNode().getNodeID();
            String host = nodeId.getHost();
            SchedulerNode node = yarnScheduler.getSchedulerNode(nodeId);
            nodeStore.add(node);
            LOGGER.info("afterSchedulerEventHandled: NM registration from node {}", host);
            break;
        case NODE_UPDATE:
            if (!(event instanceof NodeUpdateSchedulerEvent)) {
                LOGGER.error("{} not an instance of {}", event.getClass().getName(), NodeUpdateSchedulerEvent.class.getName());
                return;
            }
            RMNode rmNode = ((NodeUpdateSchedulerEvent) event).getRMNode();
            handleContainerAllocation(rmNode);
            break;
        default:
            break;
    }
}

Example 12 with SchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode in project hadoop by apache.

the class TestIncreaseAllocationExpirer method verifyAvailableResourceOfSchedulerNode.

private void verifyAvailableResourceOfSchedulerNode(MockRM rm, NodeId nodeId, int expectedMemory) {
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    SchedulerNode node = cs.getNode(nodeId);
    Assert.assertEquals(expectedMemory, node.getUnallocatedResource().getMemorySize());
}

Example 13 with SchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode in project hadoop by apache.

the class TestContainerResizing method verifyAvailableResourceOfSchedulerNode.

private void verifyAvailableResourceOfSchedulerNode(MockRM rm, NodeId nodeId, int expectedMemory) {
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    SchedulerNode node = cs.getNode(nodeId);
    Assert.assertEquals(expectedMemory, node.getUnallocatedResource().getMemorySize());
}

Example 14 with SchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode in project hadoop by apache.

the class TestFSAppAttempt method testHeadroomWithBlackListedNodes.

@Test
public void testHeadroomWithBlackListedNodes() {
    // Add two nodes
    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(8 * 1024, 8), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
    scheduler.handle(nodeEvent1);
    RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(4 * 1024, 4), 2, "127.0.0.2");
    NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
    scheduler.handle(nodeEvent2);
    assertEquals("We should have two alive nodes.", 2, scheduler.getNumClusterNodes());
    Resource clusterResource = scheduler.getClusterResource();
    Resource clusterUsage = scheduler.getRootQueueMetrics().getAllocatedResources();
    assertEquals(12 * 1024, clusterResource.getMemorySize());
    assertEquals(12, clusterResource.getVirtualCores());
    assertEquals(0, clusterUsage.getMemorySize());
    assertEquals(0, clusterUsage.getVirtualCores());
    ApplicationAttemptId id11 = createAppAttemptId(1, 1);
    createMockRMApp(id11);
    scheduler.addApplication(id11.getApplicationId(), "default", "user1", false);
    scheduler.addApplicationAttempt(id11, false, false);
    assertNotNull(scheduler.getSchedulerApplications().get(id11.getApplicationId()));
    FSAppAttempt app = scheduler.getSchedulerApp(id11);
    assertNotNull(app);
    Resource queueUsage = app.getQueue().getResourceUsage();
    assertEquals(0, queueUsage.getMemorySize());
    assertEquals(0, queueUsage.getVirtualCores());
    SchedulerNode n1 = scheduler.getSchedulerNode(node1.getNodeID());
    SchedulerNode n2 = scheduler.getSchedulerNode(node2.getNodeID());
    assertNotNull(n1);
    assertNotNull(n2);
    List<String> blacklistAdditions = new ArrayList<String>(1);
    List<String> blacklistRemovals = new ArrayList<String>(1);
    blacklistAdditions.add(n1.getNodeName());
    app.updateBlacklist(blacklistAdditions, blacklistRemovals);
    app.getQueue().setFairShare(clusterResource);
    FSAppAttempt spyApp = spy(app);
    doReturn(false).when(spyApp).isWaitingForAMContainer();
    assertTrue(spyApp.isPlaceBlacklisted(n1.getNodeName()));
    assertFalse(spyApp.isPlaceBlacklisted(n2.getNodeName()));
    assertEquals(n2.getUnallocatedResource(), spyApp.getHeadroom());
    blacklistAdditions.clear();
    blacklistAdditions.add(n2.getNodeName());
    blacklistRemovals.add(n1.getNodeName());
    app.updateBlacklist(blacklistAdditions, blacklistRemovals);
    assertFalse(spyApp.isPlaceBlacklisted(n1.getNodeName()));
    assertTrue(spyApp.isPlaceBlacklisted(n2.getNodeName()));
    assertEquals(n1.getUnallocatedResource(), spyApp.getHeadroom());
    blacklistAdditions.clear();
    blacklistRemovals.clear();
    blacklistRemovals.add(n2.getNodeName());
    app.updateBlacklist(blacklistAdditions, blacklistRemovals);
    assertFalse(spyApp.isPlaceBlacklisted(n1.getNodeName()));
    assertFalse(spyApp.isPlaceBlacklisted(n2.getNodeName()));
    assertEquals(clusterResource, spyApp.getHeadroom());
}

Example 15 with SchedulerNode

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode in project hadoop by apache.

the class TestNodeLabelContainerAllocation method testQueueMetricsWithLabels.

@Test
public void testQueueMetricsWithLabels() throws Exception {
    /**
     * Test case: have a following queue structure:
     *
     * <pre>
     *            root
     *         /      \
     *        a        b
     *        (x)     (x)
     * </pre>
     *
     * a/b can access x, both of them has max-capacity-on-x = 50
     *
     * When doing non-exclusive allocation, app in a (or b) can use 100% of x
     * resource.
     */
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration(this.conf);
    // Define top-level queues
    csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", "b" });
    csConf.setCapacityByLabel(CapacitySchedulerConfiguration.ROOT, "x", 100);
    final String queueA = CapacitySchedulerConfiguration.ROOT + ".a";
    csConf.setCapacity(queueA, 25);
    csConf.setAccessibleNodeLabels(queueA, toSet("x"));
    csConf.setCapacityByLabel(queueA, "x", 50);
    csConf.setMaximumCapacityByLabel(queueA, "x", 50);
    final String queueB = CapacitySchedulerConfiguration.ROOT + ".b";
    csConf.setCapacity(queueB, 75);
    csConf.setAccessibleNodeLabels(queueB, toSet("x"));
    csConf.setCapacityByLabel(queueB, "x", 50);
    csConf.setMaximumCapacityByLabel(queueB, "x", 50);
    // set node -> label
    mgr.addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("x", false)));
    mgr.addToCluserNodeLabels(ImmutableSet.of(NodeLabel.newInstance("y", false)));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
    // inject node label manager
    MockRM rm1 = new MockRM(csConf) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm1.getRMContext().setNodeLabelManager(mgr);
    rm1.start();
    // label = x
    MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB);
    // label = y
    MockNM nm2 = rm1.registerNode("h2:1234", 10 * GB);
    // app1 -> a
    RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a", "x");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    // app1 asks for 5 partition=x containers
    am1.allocate("*", 1 * GB, 5, new ArrayList<ContainerId>(), "x");
    // NM1 do 50 heartbeats
    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
    RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
    SchedulerNode schedulerNode1 = cs.getSchedulerNode(nm1.getNodeId());
    for (int i = 0; i < 50; i++) {
        cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
    }
    // app1 gets all resource in partition=x
    Assert.assertEquals(5, schedulerNode1.getNumContainers());
    SchedulerNodeReport reportNm1 = rm1.getResourceScheduler().getNodeReport(nm1.getNodeId());
    Assert.assertEquals(5 * GB, reportNm1.getUsedResource().getMemorySize());
    Assert.assertEquals(5 * GB, reportNm1.getAvailableResource().getMemorySize());
    SchedulerNodeReport reportNm2 = rm1.getResourceScheduler().getNodeReport(nm2.getNodeId());
    Assert.assertEquals(0 * GB, reportNm2.getUsedResource().getMemorySize());
    Assert.assertEquals(10 * GB, reportNm2.getAvailableResource().getMemorySize());
    LeafQueue leafQueue = (LeafQueue) cs.getQueue("a");
    assertEquals(0 * GB, leafQueue.getMetrics().getAvailableMB());
    assertEquals(5 * GB, leafQueue.getMetrics().getAllocatedMB());
    // Kill all apps in queue a
    cs.killAllAppsInQueue("a");
    rm1.waitForState(app1.getApplicationId(), RMAppState.KILLED);
    rm1.waitForAppRemovedFromScheduler(app1.getApplicationId());
    assertEquals(0 * GB, leafQueue.getMetrics().getUsedAMResourceMB());
    assertEquals(0, leafQueue.getMetrics().getUsedAMResourceVCores());
    rm1.close();
}