Examples with Container org.apache.hadoop.yarn.api.records.Container used on opensource projects

Example 56 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestCapacityScheduler method testApplicationHeadRoom.

// Verifies headroom passed to ApplicationMaster has been updated in
// RMAppAttemptMetrics
@Test
public void testApplicationHeadRoom() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    RMAppAttemptMetrics attemptMetric = new RMAppAttemptMetrics(appAttemptId, rm.getRMContext());
    RMAppImpl app = mock(RMAppImpl.class);
    when(app.getApplicationId()).thenReturn(appId);
    RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
    Container container = mock(Container.class);
    when(attempt.getMasterContainer()).thenReturn(container);
    ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class);
    when(attempt.getSubmissionContext()).thenReturn(submissionContext);
    when(attempt.getAppAttemptId()).thenReturn(appAttemptId);
    when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
    when(app.getCurrentAppAttempt()).thenReturn(attempt);
    rm.getRMContext().getRMApps().put(appId, app);
    SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, "default", "user");
    cs.handle(addAppEvent);
    SchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    cs.handle(addAttemptEvent);
    Allocation allocate = cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
    Assert.assertNotNull(attempt);
    Assert.assertEquals(Resource.newInstance(0, 0), allocate.getResourceLimit());
    Assert.assertEquals(Resource.newInstance(0, 0), attemptMetric.getApplicationAttemptHeadroom());
    // Add a node to cluster
    Resource newResource = Resource.newInstance(4 * GB, 1);
    RMNode node = MockNodes.newNodeInfo(0, newResource, 1, "127.0.0.1");
    cs.handle(new NodeAddedSchedulerEvent(node));
    allocate = cs.allocate(appAttemptId, Collections.<ResourceRequest>emptyList(), Collections.<ContainerId>emptyList(), null, null, NULL_UPDATE_REQUESTS);
    // All resources should be sent as headroom
    Assert.assertEquals(newResource, allocate.getResourceLimit());
    Assert.assertEquals(newResource, attemptMetric.getApplicationAttemptHeadroom());
    rm.stop();
}

Example 57 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestCapacityScheduler method testPreemptionInfo.

@Test(timeout = 120000)
public void testPreemptionInfo() throws Exception {
    Configuration conf = new Configuration();
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 3);
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    // start RM
    int CONTAINER_MEMORY = 1024;
    MockRM rm1 = new MockRM(conf);
    rm1.start();
    // get scheduler
    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
    // start NM
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // create app and launch the AM
    RMApp app0 = rm1.submitApp(CONTAINER_MEMORY);
    MockAM am0 = MockRM.launchAM(app0, rm1, nm1);
    am0.registerAppAttempt();
    // get scheduler app
    FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications().get(app0.getApplicationId()).getCurrentAppAttempt();
    // allocate some containers and launch them
    List<Container> allocatedContainers = am0.allocateAndWaitForContainers(3, CONTAINER_MEMORY, nm1);
    // kill the 3 containers
    for (Container c : allocatedContainers) {
        cs.markContainerForKillable(schedulerAppAttempt.getRMContainer(c.getId()));
    }
    // check values
    waitForAppPreemptionInfo(app0, Resource.newInstance(CONTAINER_MEMORY * 3, 3), 0, 3, Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3);
    // kill app0-attempt0 AM container
    cs.markContainerForKillable(schedulerAppAttempt.getRMContainer(app0.getCurrentAppAttempt().getMasterContainer().getId()));
    // wait for app0 failed
    waitForNewAttemptCreated(app0, am0.getApplicationAttemptId());
    // check values
    waitForAppPreemptionInfo(app0, Resource.newInstance(CONTAINER_MEMORY * 4, 4), 1, 3, Resource.newInstance(0, 0), false, 0);
    // launch app0-attempt1
    MockAM am1 = MockRM.launchAM(app0, rm1, nm1);
    am1.registerAppAttempt();
    schedulerAppAttempt = cs.getSchedulerApplications().get(app0.getApplicationId()).getCurrentAppAttempt();
    // allocate some containers and launch them
    allocatedContainers = am1.allocateAndWaitForContainers(3, CONTAINER_MEMORY, nm1);
    for (Container c : allocatedContainers) {
        cs.markContainerForKillable(schedulerAppAttempt.getRMContainer(c.getId()));
    }
    // check values
    waitForAppPreemptionInfo(app0, Resource.newInstance(CONTAINER_MEMORY * 7, 7), 1, 6, Resource.newInstance(CONTAINER_MEMORY * 3, 3), false, 3);
    rm1.stop();
}

Example 58 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestCapacityScheduler method testMoveAttemptNotAdded.

@Test(timeout = 60000)
public void testMoveAttemptNotAdded() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(getCapacityConfiguration(conf));
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    ApplicationId appId = BuilderUtils.newApplicationId(100, 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    RMAppAttemptMetrics attemptMetric = new RMAppAttemptMetrics(appAttemptId, rm.getRMContext());
    RMAppImpl app = mock(RMAppImpl.class);
    when(app.getApplicationId()).thenReturn(appId);
    RMAppAttemptImpl attempt = mock(RMAppAttemptImpl.class);
    Container container = mock(Container.class);
    when(attempt.getMasterContainer()).thenReturn(container);
    ApplicationSubmissionContext submissionContext = mock(ApplicationSubmissionContext.class);
    when(attempt.getSubmissionContext()).thenReturn(submissionContext);
    when(attempt.getAppAttemptId()).thenReturn(appAttemptId);
    when(attempt.getRMAppAttemptMetrics()).thenReturn(attemptMetric);
    when(app.getCurrentAppAttempt()).thenReturn(attempt);
    rm.getRMContext().getRMApps().put(appId, app);
    SchedulerEvent addAppEvent = new AppAddedSchedulerEvent(appId, "a1", "user");
    try {
        cs.moveApplication(appId, "b1");
        fail("Move should throw exception app not available");
    } catch (YarnException e) {
        assertEquals("App to be moved application_100_0001 not found.", e.getMessage());
    }
    cs.handle(addAppEvent);
    cs.moveApplication(appId, "b1");
    SchedulerEvent addAttemptEvent = new AppAttemptAddedSchedulerEvent(appAttemptId, false);
    cs.handle(addAttemptEvent);
    CSQueue rootQ = cs.getRootQueue();
    CSQueue queueB = cs.getQueue("b");
    CSQueue queueA = cs.getQueue("a");
    CSQueue queueA1 = cs.getQueue("a1");
    CSQueue queueB1 = cs.getQueue("b1");
    Assert.assertEquals(1, rootQ.getNumApplications());
    Assert.assertEquals(0, queueA.getNumApplications());
    Assert.assertEquals(1, queueB.getNumApplications());
    Assert.assertEquals(0, queueA1.getNumApplications());
    Assert.assertEquals(1, queueB1.getNumApplications());
    rm.close();
}

Example 59 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestSchedulingWithAllocationRequestId method testMultipleAllocationRequestIds.

@Test
public void testMultipleAllocationRequestIds() throws Exception {
    configureScheduler();
    YarnConfiguration conf = getConf();
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB);
        MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB);
        RMApp app1 = rm.submitApp(2048);
        // kick the scheduling
        nm1.nodeHeartbeat(true);
        RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
        MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
        am1.registerAppAttempt();
        // add request for containers with id 10 & 20
        am1.addRequests(new String[] { "127.0.0.1" }, 2 * GB, 1, 1, 10L);
        // send the request
        AllocateResponse allocResponse = am1.schedule();
        am1.addRequests(new String[] { "127.0.0.2" }, 2 * GB, 1, 2, 20L);
        // send the request
        allocResponse = am1.schedule();
        // check if request id 10 is satisfied
        nm1.nodeHeartbeat(true);
        // send the request
        allocResponse = am1.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        List<Container> allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 2 * GB, nm1.getNodeId(), 10);
        // check now if request id 20 is satisfied
        nm2.nodeHeartbeat(true);
        while (allocResponse.getAllocatedContainers().size() < 2) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(2, allocated.size());
        for (Container container : allocated) {
            checkAllocatedContainer(container, 2 * GB, nm2.getNodeId(), 20);
        }
    } finally {
        if (rm != null) {
            rm.stop();
        }
    }
}

Example 60 with Container

use of org.apache.hadoop.yarn.api.records.Container in project hadoop by apache.

the class TestSchedulingWithAllocationRequestId method testMultipleAppsWithAllocationReqId.

@Test
public void testMultipleAppsWithAllocationReqId() throws Exception {
    configureScheduler();
    YarnConfiguration conf = getConf();
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        // Register node1
        String host0 = "host_0";
        String host1 = "host_1";
        MockNM nm1 = new MockNM(host0 + ":1234", 8 * GB, rm.getResourceTrackerService());
        nm1.registerNode();
        // Register node2
        MockNM nm2 = new MockNM(host1 + ":2351", 8 * GB, rm.getResourceTrackerService());
        nm2.registerNode();
        // submit 1st app
        RMApp app1 = rm.submitApp(1 * GB, "user_0", "a1");
        MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
        // Submit app1 RR with allocationReqId = 5
        int numContainers = 1;
        am1.addRequests(new String[] { host0, host1 }, 1 * GB, 1, numContainers, 5L);
        AllocateResponse allocResponse = am1.schedule();
        // wait for containers to be allocated.
        nm1.nodeHeartbeat(true);
        // send the request
        allocResponse = am1.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        List<Container> allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 1 * GB, nm1.getNodeId(), 5L);
        // Submit another application
        RMApp app2 = rm.submitApp(1 * GB, "user_1", "a2");
        MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2);
        // Submit app2 RR with allocationReqId = 5
        am2.addRequests(new String[] { host0, host1 }, 2 * GB, 1, numContainers, 5L);
        am2.schedule();
        // wait for containers to be allocated.
        nm2.nodeHeartbeat(true);
        // send the request
        allocResponse = am2.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am2.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 2 * GB, nm2.getNodeId(), 5L);
        // Now submit app2 RR with allocationReqId = 10
        am2.addRequests(new String[] { host0, host1 }, 3 * GB, 1, numContainers, 10L);
        am2.schedule();
        // wait for containers to be allocated.
        nm1.nodeHeartbeat(true);
        // send the request
        allocResponse = am2.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am2.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 3 * GB, nm1.getNodeId(), 10L);
        // Now submit app1 RR with allocationReqId = 10
        am1.addRequests(new String[] { host0, host1 }, 4 * GB, 1, numContainers, 10L);
        am1.schedule();
        // wait for containers to be allocated.
        nm2.nodeHeartbeat(true);
        // send the request
        allocResponse = am1.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 4 * GB, nm2.getNodeId(), 10L);
    } finally {
        if (rm != null) {
            rm.stop();
        }
    }
}