Examples with NodeUpdateSchedulerEvent org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent used on opensource projects

Example 11 with NodeUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent in project hadoop by apache.

the class TestFairScheduler method testReservationWithMultiplePriorities.

/**
   * Reserve at a lower priority and verify the lower priority request gets
   * allocated
   */
@Test(timeout = 5000)
public void testReservationWithMultiplePriorities() throws IOException {
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    // Add a node
    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2));
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
    scheduler.handle(nodeEvent1);
    NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
    // Create first app and take up half resources so the second app that asks
    // for the entire node won't have enough.
    FSAppAttempt app1 = scheduler.getSchedulerApp(createSchedulingRequest(1024, 1, "queue", "user", 1));
    scheduler.update();
    scheduler.handle(updateEvent);
    assertEquals("Basic allocation failed", 1, app1.getLiveContainers().size());
    // Create another app and reserve at a lower priority first
    ApplicationAttemptId attId = createSchedulingRequest(2048, 2, "queue1", "user1", 1, 2);
    FSAppAttempt app2 = scheduler.getSchedulerApp(attId);
    scheduler.update();
    scheduler.handle(updateEvent);
    assertEquals("Reservation at lower priority failed", 1, app2.getReservedContainers().size());
    // Request container on the second app at a higher priority
    createSchedulingRequestExistingApplication(2048, 2, 1, attId);
    // Complete the first container so we can trigger allocation for app2
    ContainerId containerId = app1.getLiveContainers().iterator().next().getContainerId();
    scheduler.allocate(app1.getApplicationAttemptId(), new ArrayList<>(), Arrays.asList(containerId), null, null, NULL_UPDATE_REQUESTS);
    // Trigger allocation for app2
    scheduler.handle(updateEvent);
    // Reserved container (at lower priority) should be run
    Collection<RMContainer> liveContainers = app2.getLiveContainers();
    assertEquals("Allocation post completion failed", 1, liveContainers.size());
    assertEquals("High prio container allocated against low prio reservation", 2, liveContainers.iterator().next().getContainer().getPriority().getPriority());
}

Example 12 with NodeUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent in project hadoop by apache.

the class TestFairScheduler method testContainerAllocationWithContainerIdLeap.

@Test(timeout = 120000)
public void testContainerAllocationWithContainerIdLeap() throws Exception {
    conf.setFloat(FairSchedulerConfiguration.RESERVABLE_NODES, 0.50f);
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    // Add two node
    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(3072, 10), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
    scheduler.handle(nodeEvent1);
    RMNode node2 = MockNodes.newNodeInfo(1, Resources.createResource(3072, 10), 1, "127.0.0.2");
    NodeAddedSchedulerEvent nodeEvent2 = new NodeAddedSchedulerEvent(node2);
    scheduler.handle(nodeEvent2);
    ApplicationAttemptId app1 = createSchedulingRequest(2048, "queue1", "user1", 2);
    scheduler.update();
    scheduler.handle(new NodeUpdateSchedulerEvent(node1));
    scheduler.handle(new NodeUpdateSchedulerEvent(node2));
    ApplicationAttemptId app2 = createSchedulingRequest(2048, "queue1", "user1", 1);
    scheduler.update();
    scheduler.handle(new NodeUpdateSchedulerEvent(node1));
    scheduler.handle(new NodeUpdateSchedulerEvent(node2));
    assertEquals(4096, scheduler.getQueueManager().getQueue("queue1").getResourceUsage().getMemorySize());
    //container will be reserved at node1
    RMContainer reservedContainer1 = scheduler.getSchedulerNode(node1.getNodeID()).getReservedContainer();
    assertNotEquals(reservedContainer1, null);
    RMContainer reservedContainer2 = scheduler.getSchedulerNode(node2.getNodeID()).getReservedContainer();
    assertEquals(reservedContainer2, null);
    for (int i = 0; i < 10; i++) {
        scheduler.handle(new NodeUpdateSchedulerEvent(node1));
        scheduler.handle(new NodeUpdateSchedulerEvent(node2));
    }
    // release resource
    scheduler.handle(new AppAttemptRemovedSchedulerEvent(app1, RMAppAttemptState.KILLED, false));
    assertEquals(0, scheduler.getQueueManager().getQueue("queue1").getResourceUsage().getMemorySize());
    // container will be allocated at node2
    scheduler.handle(new NodeUpdateSchedulerEvent(node2));
    assertEquals(scheduler.getSchedulerApp(app2).getLiveContainers().size(), 1);
    long maxId = 0;
    for (RMContainer container : scheduler.getSchedulerApp(app2).getLiveContainers()) {
        assertTrue(container.getContainer().getNodeId().equals(node2.getNodeID()));
        if (container.getContainerId().getContainerId() > maxId) {
            maxId = container.getContainerId().getContainerId();
        }
    }
    long reservedId = reservedContainer1.getContainerId().getContainerId();
    assertEquals(reservedId + 1, maxId);
}

Example 13 with NodeUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent in project hadoop by apache.

the class TestFairScheduler method testRequestAMResourceInZeroFairShareQueue.

/**
   * The test verifies that zero-FairShare queues (because of zero/tiny
   * weight) can get resources for the AM.
   */
@Test
public void testRequestAMResourceInZeroFairShareQueue() throws Exception {
    conf.set(FairSchedulerConfiguration.ALLOCATION_FILE, ALLOC_FILE);
    PrintWriter out = new PrintWriter(new FileWriter(ALLOC_FILE));
    out.println("<?xml version=\"1.0\"?>");
    out.println("<allocations>");
    out.println("<queue name=\"queue1\">");
    out.println("<weight>0.0</weight>");
    out.println("<maxResources>4096mb,10vcores</maxResources>");
    out.println("<maxAMShare>0.5</maxAMShare>");
    out.println("</queue>");
    out.println("<queue name=\"queue2\">");
    out.println("<weight>2.0</weight>");
    out.println("</queue>");
    out.println("<queue name=\"queue3\">");
    out.println("<weight>0.000001</weight>");
    out.println("</queue>");
    out.println("</allocations>");
    out.close();
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(8192, 20), 0, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
    NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
    scheduler.handle(nodeEvent);
    scheduler.update();
    //create request for non-zero weight queue
    createSchedulingRequest(1024, "root.queue2", "user2", 1);
    scheduler.update();
    scheduler.handle(updateEvent);
    // A managed AM which need 3G memory will not get resource,
    // since it request more than the maxAMShare (4G * 0.5 = 2G).
    Resource amResource = Resource.newInstance(1024, 1);
    int amPriority = RMAppAttemptImpl.AM_CONTAINER_PRIORITY.getPriority();
    ApplicationAttemptId attId1 = createAppAttemptId(1, 1);
    createApplicationWithAMResource(attId1, "root.queue1", "user1", amResource);
    createSchedulingRequestExistingApplication(3 * 1024, 1, amPriority, attId1);
    FSAppAttempt app1 = scheduler.getSchedulerApp(attId1);
    scheduler.update();
    scheduler.handle(updateEvent);
    assertEquals("Application 1 should not be running", 0, app1.getLiveContainers().size());
    // A managed AM which need 2G memory will get resource,
    // since it request no more than the maxAMShare (4G * 0.5 = 2G).
    ApplicationAttemptId attId2 = createAppAttemptId(2, 1);
    createApplicationWithAMResource(attId2, "root.queue1", "user1", amResource);
    createSchedulingRequestExistingApplication(2 * 1024, 1, amPriority, attId2);
    FSAppAttempt app2 = scheduler.getSchedulerApp(attId2);
    scheduler.update();
    scheduler.handle(updateEvent);
    assertEquals("Application 2 should be running", 1, app2.getLiveContainers().size());
    // A managed AM which need 1G memory will get resource, even thought its
    // fair share is 0 because its weight is tiny(0.000001).
    ApplicationAttemptId attId3 = createAppAttemptId(3, 1);
    createApplicationWithAMResource(attId3, "root.queue3", "user1", amResource);
    createSchedulingRequestExistingApplication(1024, 1, amPriority, attId3);
    FSAppAttempt app3 = scheduler.getSchedulerApp(attId3);
    scheduler.update();
    scheduler.handle(updateEvent);
    assertEquals("Application 3 should be running", 1, app3.getLiveContainers().size());
}

Example 14 with NodeUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent in project hadoop by apache.

the class TestFairScheduler method testMoveWouldViolateMaxResourcesConstraints.

@Test(expected = YarnException.class)
public void testMoveWouldViolateMaxResourcesConstraints() throws Exception {
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    QueueManager queueMgr = scheduler.getQueueManager();
    FSLeafQueue oldQueue = queueMgr.getLeafQueue("queue1", true);
    FSQueue queue2 = queueMgr.getLeafQueue("queue2", true);
    queue2.setMaxShare(Resource.newInstance(1024, 1));
    ApplicationAttemptId appAttId = createSchedulingRequest(1024, 1, "queue1", "user1", 3);
    RMNode node = MockNodes.newNodeInfo(1, Resources.createResource(2048, 2));
    NodeAddedSchedulerEvent nodeEvent = new NodeAddedSchedulerEvent(node);
    NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node);
    scheduler.handle(nodeEvent);
    scheduler.handle(updateEvent);
    scheduler.handle(updateEvent);
    assertEquals(Resource.newInstance(2048, 2), oldQueue.getResourceUsage());
    scheduler.moveApplication(appAttId.getApplicationId(), "queue2");
}

Example 15 with NodeUpdateSchedulerEvent

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent in project hadoop by apache.

the class TestFairScheduler method testNoMoreCpuOnNode.

@Test
public void testNoMoreCpuOnNode() throws IOException {
    scheduler.init(conf);
    scheduler.start();
    scheduler.reinitialize(conf, resourceManager.getRMContext());
    RMNode node1 = MockNodes.newNodeInfo(1, Resources.createResource(2048, 1), 1, "127.0.0.1");
    NodeAddedSchedulerEvent nodeEvent1 = new NodeAddedSchedulerEvent(node1);
    scheduler.handle(nodeEvent1);
    ApplicationAttemptId attId = createSchedulingRequest(1024, 1, "default", "user1", 2);
    FSAppAttempt app = scheduler.getSchedulerApp(attId);
    scheduler.update();
    NodeUpdateSchedulerEvent updateEvent = new NodeUpdateSchedulerEvent(node1);
    scheduler.handle(updateEvent);
    assertEquals(1, app.getLiveContainers().size());
    scheduler.handle(updateEvent);
    assertEquals(1, app.getLiveContainers().size());
}