Examples with NodeAddedSchedulerEvent org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent used on opensource projects

Example 36 with NodeAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent in project hadoop by apache.

the class TestSchedulerHealth method registerNode.

private NodeManager registerNode(String hostName, int containerManagerPort, int httpPort, String rackName, Resource capability) throws IOException, YarnException {
    NodeManager nm = new NodeManager(hostName, containerManagerPort, httpPort, rackName, capability, resourceManager);
    NodeAddedSchedulerEvent nodeAddEvent1 = new NodeAddedSchedulerEvent(resourceManager.getRMContext().getRMNodes().get(nm.getNodeId()));
    resourceManager.getResourceScheduler().handle(nodeAddEvent1);
    return nm;
}

Example 37 with NodeAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent in project hadoop by apache.

the class TestCapacityScheduler method testNumClusterNodes.

@Test
public void testNumClusterNodes() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    CapacityScheduler cs = new CapacityScheduler();
    cs.setConf(conf);
    RMContext rmContext = TestUtils.getMockRMContext();
    cs.setRMContext(rmContext);
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    setupQueueConfiguration(csConf);
    cs.init(csConf);
    cs.start();
    assertEquals(0, cs.getNumClusterNodes());
    RMNode n1 = MockNodes.newNodeInfo(0, MockNodes.newResource(4 * GB), 1);
    RMNode n2 = MockNodes.newNodeInfo(0, MockNodes.newResource(2 * GB), 2);
    cs.handle(new NodeAddedSchedulerEvent(n1));
    cs.handle(new NodeAddedSchedulerEvent(n2));
    assertEquals(2, cs.getNumClusterNodes());
    cs.handle(new NodeRemovedSchedulerEvent(n1));
    assertEquals(1, cs.getNumClusterNodes());
    cs.handle(new NodeAddedSchedulerEvent(n1));
    assertEquals(2, cs.getNumClusterNodes());
    cs.handle(new NodeRemovedSchedulerEvent(n2));
    cs.handle(new NodeRemovedSchedulerEvent(n1));
    assertEquals(0, cs.getNumClusterNodes());
    cs.stop();
}

Example 38 with NodeAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent in project hadoop by apache.

the class TestAbstractYarnScheduler method testMaxAllocationAfterUpdateNodeResource.

@Test
public void testMaxAllocationAfterUpdateNodeResource() throws IOException {
    final int configuredMaxVCores = 20;
    final int configuredMaxMemory = 10 * 1024;
    Resource configuredMaximumResource = Resource.newInstance(configuredMaxMemory, configuredMaxVCores);
    YarnConfiguration conf = getConf();
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_VCORES, configuredMaxVCores);
    conf.setInt(YarnConfiguration.RM_SCHEDULER_MAXIMUM_ALLOCATION_MB, configuredMaxMemory);
    conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0);
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        AbstractYarnScheduler scheduler = (AbstractYarnScheduler) rm.getResourceScheduler();
        verifyMaximumResourceCapability(configuredMaximumResource, scheduler);
        Resource resource1 = Resource.newInstance(2048, 5);
        Resource resource2 = Resource.newInstance(4096, 10);
        Resource resource3 = Resource.newInstance(512, 1);
        Resource resource4 = Resource.newInstance(1024, 2);
        RMNode node1 = MockNodes.newNodeInfo(0, resource1, 1, "127.0.0.2");
        scheduler.handle(new NodeAddedSchedulerEvent(node1));
        RMNode node2 = MockNodes.newNodeInfo(0, resource3, 2, "127.0.0.3");
        scheduler.handle(new NodeAddedSchedulerEvent(node2));
        verifyMaximumResourceCapability(resource1, scheduler);
        // increase node1 resource
        scheduler.updateNodeResource(node1, ResourceOption.newInstance(resource2, 0));
        verifyMaximumResourceCapability(resource2, scheduler);
        // decrease node1 resource
        scheduler.updateNodeResource(node1, ResourceOption.newInstance(resource1, 0));
        verifyMaximumResourceCapability(resource1, scheduler);
        // increase node2 resource
        scheduler.updateNodeResource(node2, ResourceOption.newInstance(resource4, 0));
        verifyMaximumResourceCapability(resource1, scheduler);
        // decrease node2 resource
        scheduler.updateNodeResource(node2, ResourceOption.newInstance(resource3, 0));
        verifyMaximumResourceCapability(resource1, scheduler);
    } finally {
        rm.stop();
    }
}

Example 39 with NodeAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent in project hadoop by apache.

the class TestAbstractYarnScheduler method testMaximumAllocationMemoryHelper.

private void testMaximumAllocationMemoryHelper(YarnScheduler scheduler, final int node1MaxMemory, final int node2MaxMemory, final int node3MaxMemory, final int... expectedMaxMemory) throws Exception {
    Assert.assertEquals(6, expectedMaxMemory.length);
    Assert.assertEquals(0, scheduler.getNumClusterNodes());
    long maxMemory = scheduler.getMaximumResourceCapability().getMemorySize();
    Assert.assertEquals(expectedMaxMemory[0], maxMemory);
    RMNode node1 = MockNodes.newNodeInfo(0, Resources.createResource(node1MaxMemory), 1, "127.0.0.2");
    scheduler.handle(new NodeAddedSchedulerEvent(node1));
    Assert.assertEquals(1, scheduler.getNumClusterNodes());
    maxMemory = scheduler.getMaximumResourceCapability().getMemorySize();
    Assert.assertEquals(expectedMaxMemory[1], maxMemory);
    scheduler.handle(new NodeRemovedSchedulerEvent(node1));
    Assert.assertEquals(0, scheduler.getNumClusterNodes());
    maxMemory = scheduler.getMaximumResourceCapability().getMemorySize();
    Assert.assertEquals(expectedMaxMemory[2], maxMemory);
    RMNode node2 = MockNodes.newNodeInfo(0, Resources.createResource(node2MaxMemory), 2, "127.0.0.3");
    scheduler.handle(new NodeAddedSchedulerEvent(node2));
    Assert.assertEquals(1, scheduler.getNumClusterNodes());
    maxMemory = scheduler.getMaximumResourceCapability().getMemorySize();
    Assert.assertEquals(expectedMaxMemory[3], maxMemory);
    RMNode node3 = MockNodes.newNodeInfo(0, Resources.createResource(node3MaxMemory), 3, "127.0.0.4");
    scheduler.handle(new NodeAddedSchedulerEvent(node3));
    Assert.assertEquals(2, scheduler.getNumClusterNodes());
    maxMemory = scheduler.getMaximumResourceCapability().getMemorySize();
    Assert.assertEquals(expectedMaxMemory[4], maxMemory);
    scheduler.handle(new NodeRemovedSchedulerEvent(node3));
    Assert.assertEquals(1, scheduler.getNumClusterNodes());
    maxMemory = scheduler.getMaximumResourceCapability().getMemorySize();
    Assert.assertEquals(expectedMaxMemory[5], maxMemory);
    scheduler.handle(new NodeRemovedSchedulerEvent(node2));
    Assert.assertEquals(0, scheduler.getNumClusterNodes());
}

Example 40 with NodeAddedSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent in project hadoop by apache.

the class TestOpportunisticContainerAllocatorAMService method testNodeRemovalDuringAllocate.

@Test(timeout = 60000)
public void testNodeRemovalDuringAllocate() throws Exception {
    MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
    MockNM nm2 = new MockNM("h2:1234", 4096, rm.getResourceTrackerService());
    nm1.registerNode();
    nm2.registerNode();
    OpportunisticContainerAllocatorAMService amservice = (OpportunisticContainerAllocatorAMService) rm.getApplicationMasterService();
    RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
    ApplicationAttemptId attemptId = app1.getCurrentAppAttempt().getAppAttemptId();
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm2);
    ResourceScheduler scheduler = rm.getResourceScheduler();
    RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
    RMNode rmNode2 = rm.getRMContext().getRMNodes().get(nm2.getNodeId());
    nm1.nodeHeartbeat(true);
    nm2.nodeHeartbeat(true);
    ((RMNodeImpl) rmNode1).setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
    ((RMNodeImpl) rmNode2).setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
    OpportunisticContainerContext ctxt = ((CapacityScheduler) scheduler).getApplicationAttempt(attemptId).getOpportunisticContainerContext();
    // Send add and update node events to AM Service.
    amservice.handle(new NodeAddedSchedulerEvent(rmNode1));
    amservice.handle(new NodeAddedSchedulerEvent(rmNode2));
    amservice.handle(new NodeUpdateSchedulerEvent(rmNode1));
    amservice.handle(new NodeUpdateSchedulerEvent(rmNode2));
    // Both node 1 and node 2 will be applicable for scheduling.
    for (int i = 0; i < 10; i++) {
        am1.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*", Resources.createResource(1 * GB), 2)), null);
        if (ctxt.getNodeMap().size() == 2) {
            break;
        }
        Thread.sleep(50);
    }
    Assert.assertEquals(2, ctxt.getNodeMap().size());
    // Remove node from scheduler but not from AM Service.
    scheduler.handle(new NodeRemovedSchedulerEvent(rmNode1));
    // After removal of node 1, only 1 node will be applicable for scheduling.
    for (int i = 0; i < 10; i++) {
        try {
            am1.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*", Resources.createResource(1 * GB), 2)), null);
        } catch (Exception e) {
            Assert.fail("Allocate request should be handled on node removal");
        }
        if (ctxt.getNodeMap().size() == 1) {
            break;
        }
        Thread.sleep(50);
    }
    Assert.assertEquals(1, ctxt.getNodeMap().size());
}