Examples with MockAM org.apache.hadoop.yarn.server.resourcemanager.MockAM used on opensource projects

Example 51 with MockAM

use of org.apache.hadoop.yarn.server.resourcemanager.MockAM in project hadoop by apache.

the class TestCapacityScheduler method testParentQueueMaxCapsAreRespected.

@Test
public void testParentQueueMaxCapsAreRespected() throws Exception {
    /*
     * Queue tree:
     *          Root
     *        /     \
     *       A       B
     *      / \
     *     A1 A2 
     */
    CapacitySchedulerConfiguration csConf = new CapacitySchedulerConfiguration();
    csConf.setQueues(CapacitySchedulerConfiguration.ROOT, new String[] { "a", "b" });
    csConf.setCapacity(A, 50);
    csConf.setMaximumCapacity(A, 50);
    csConf.setCapacity(B, 50);
    // Define 2nd-level queues
    csConf.setQueues(A, new String[] { "a1", "a2" });
    csConf.setCapacity(A1, 50);
    csConf.setUserLimitFactor(A1, 100.0f);
    csConf.setCapacity(A2, 50);
    csConf.setUserLimitFactor(A2, 100.0f);
    csConf.setCapacity(B1, B1_CAPACITY);
    csConf.setUserLimitFactor(B1, 100.0f);
    YarnConfiguration conf = new YarnConfiguration(csConf);
    conf.setBoolean(CapacitySchedulerConfiguration.ENABLE_USER_METRICS, true);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    MockRM rm1 = new MockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 24 * GB, rm1.getResourceTrackerService());
    nm1.registerNode();
    // Launch app1 in a1, resource usage is 1GB (am) + 4GB * 2 = 9GB 
    RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a1");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    waitContainerAllocated(am1, 4 * GB, 2, 2, rm1, nm1);
    // Try to launch app2 in a2, asked 2GB, should success 
    RMApp app2 = rm1.submitApp(2 * GB, "app", "user", null, "a2");
    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
    try {
        // Try to allocate a container, a's usage=11G/max=12
        // a1's usage=9G/max=12
        // a2's usage=2G/max=12
        // In this case, if a2 asked 2G, should fail.
        waitContainerAllocated(am2, 2 * GB, 1, 2, rm1, nm1);
    } catch (AssertionError failure) {
        // Expected, return;
        return;
    }
    Assert.fail("Shouldn't successfully allocate containers for am2, " + "queue-a's max capacity will be violated if container allocated");
}

Example 52 with MockAM

use of org.apache.hadoop.yarn.server.resourcemanager.MockAM in project hadoop by apache.

the class TestCapacitySchedulerAsyncScheduling method testAsyncContainerAllocation.

public void testAsyncContainerAllocation(int numThreads) throws Exception {
    conf.setInt(CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_MAXIMUM_THREAD, numThreads);
    conf.setInt(CapacitySchedulerConfiguration.SCHEDULE_ASYNCHRONOUSLY_PREFIX + ".scheduling-interval-ms", 100);
    final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    // inject node label manager
    MockRM rm = new MockRM(TestUtils.getConfigurationWithMultipleQueues(conf)) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    List<MockNM> nms = new ArrayList<>();
    // Add 10 nodes to the cluster, in the cluster we have 200 GB resource
    for (int i = 0; i < 10; i++) {
        nms.add(rm.registerNode("h-" + i + ":1234", 20 * GB));
    }
    List<MockAM> ams = new ArrayList<>();
    // Add 3 applications to the cluster, one app in one queue
    // the i-th app ask (20 * i) containers. So in total we will have
    // 123G container allocated
    // 3 AMs
    int totalAsked = 3 * GB;
    for (int i = 0; i < 3; i++) {
        RMApp rmApp = rm.submitApp(1024, "app", "user", null, false, Character.toString((char) (i % 34 + 97)), 1, null, null, false);
        MockAM am = MockRM.launchAMWhenAsyncSchedulingEnabled(rmApp, rm);
        am.registerAppAttempt();
        ams.add(am);
    }
    for (int i = 0; i < 3; i++) {
        ams.get(i).allocate("*", 1024, 20 * (i + 1), new ArrayList<>());
        totalAsked += 20 * (i + 1) * GB;
    }
    // Wait for at most 15000 ms
    // ms
    int waitTime = 15000;
    while (waitTime > 0) {
        if (rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB() == totalAsked) {
            break;
        }
        Thread.sleep(50);
        waitTime -= 50;
    }
    Assert.assertEquals(rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB(), totalAsked);
    // Wait for another 2 sec to make sure we will not allocate more than
    // required
    // ms
    waitTime = 2000;
    while (waitTime > 0) {
        Assert.assertEquals(rm.getResourceScheduler().getRootQueueMetrics().getAllocatedMB(), totalAsked);
        waitTime -= 50;
        Thread.sleep(50);
    }
    rm.close();
}

Example 53 with MockAM

use of org.apache.hadoop.yarn.server.resourcemanager.MockAM in project hadoop by apache.

the class TestCapacitySchedulerLazyPreemption method testSimplePreemption.

@Test(timeout = 60000)
public void testSimplePreemption() throws Exception {
    /**
     * Test case: Submit two application (app1/app2) to different queues, queue
     * structure:
     *
     * <pre>
     *             Root
     *            /  |  \
     *           a   b   c
     *          10   20  70
     * </pre>
     *
     * 1) Two nodes in the cluster, each of them has 4G.
     *
     * 2) app1 submit to queue-a first, it asked 7 * 1G containers, so there's no
     * more resource available.
     *
     * 3) app2 submit to queue-c, ask for one 1G container (for AM)
     *
     * Now the cluster is fulfilled.
     *
     * 4) app2 asks for another 1G container, system will preempt one container
     * from app1, and app2 will receive the preempted container
     */
    MockRM rm1 = new MockRM(conf);
    rm1.getRMContext().setNodeLabelManager(mgr);
    rm1.start();
    MockNM nm1 = rm1.registerNode("h1:1234", 4 * GB);
    MockNM nm2 = rm1.registerNode("h2:1234", 4 * GB);
    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
    RMNode rmNode1 = rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
    RMNode rmNode2 = rm1.getRMContext().getRMNodes().get(nm2.getNodeId());
    // launch an app to queue, AM container should be launched in nm1
    RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "a");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    am1.allocate("*", 1 * GB, 7, new ArrayList<ContainerId>());
    // Do allocation 3 times for node1/node2
    for (int i = 0; i < 3; i++) {
        cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
        cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
    }
    // App1 should have 7 containers now, and no available resource for cluster
    FiCaSchedulerApp schedulerApp1 = cs.getApplicationAttempt(am1.getApplicationAttemptId());
    Assert.assertEquals(7, schedulerApp1.getLiveContainers().size());
    // Submit app2 to queue-c and asks for a 1G container for AM
    RMApp app2 = rm1.submitApp(1 * GB, "app", "user", null, "c");
    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm2);
    // NM1/NM2 has available resource = 0G
    Assert.assertEquals(0 * GB, cs.getNode(nm1.getNodeId()).getUnallocatedResource().getMemorySize());
    Assert.assertEquals(0 * GB, cs.getNode(nm2.getNodeId()).getUnallocatedResource().getMemorySize());
    // AM asks for a 1 * GB container
    am2.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), ResourceRequest.ANY, Resources.createResource(1 * GB), 1)), null);
    // Get edit policy and do one update
    SchedulingEditPolicy editPolicy = getSchedulingEditPolicy(rm1);
    // Call edit schedule twice, and check if one container from app1 marked
    // to be "killable"
    editPolicy.editSchedule();
    editPolicy.editSchedule();
    PreemptionManager pm = cs.getPreemptionManager();
    Map<ContainerId, RMContainer> killableContainers = waitKillableContainersSize(pm, "a", RMNodeLabelsManager.NO_LABEL, 1);
    Assert.assertEquals(1, killableContainers.size());
    Assert.assertEquals(killableContainers.entrySet().iterator().next().getKey().getApplicationAttemptId(), am1.getApplicationAttemptId());
    // Call CS.handle once to see if container preempted
    cs.handle(new NodeUpdateSchedulerEvent(rmNode2));
    FiCaSchedulerApp schedulerApp2 = cs.getApplicationAttempt(am2.getApplicationAttemptId());
    // App1 has 6 containers, and app2 has 2 containers
    Assert.assertEquals(6, schedulerApp1.getLiveContainers().size());
    Assert.assertEquals(2, schedulerApp2.getLiveContainers().size());
    rm1.close();
}

Example 54 with MockAM

use of org.apache.hadoop.yarn.server.resourcemanager.MockAM in project hadoop by apache.

the class TestSchedulingWithAllocationRequestId method testMultipleAllocationRequestIds.

@Test
public void testMultipleAllocationRequestIds() throws Exception {
    configureScheduler();
    YarnConfiguration conf = getConf();
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB);
        MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB);
        RMApp app1 = rm.submitApp(2048);
        // kick the scheduling
        nm1.nodeHeartbeat(true);
        RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
        MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
        am1.registerAppAttempt();
        // add request for containers with id 10 & 20
        am1.addRequests(new String[] { "127.0.0.1" }, 2 * GB, 1, 1, 10L);
        // send the request
        AllocateResponse allocResponse = am1.schedule();
        am1.addRequests(new String[] { "127.0.0.2" }, 2 * GB, 1, 2, 20L);
        // send the request
        allocResponse = am1.schedule();
        // check if request id 10 is satisfied
        nm1.nodeHeartbeat(true);
        // send the request
        allocResponse = am1.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        List<Container> allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 2 * GB, nm1.getNodeId(), 10);
        // check now if request id 20 is satisfied
        nm2.nodeHeartbeat(true);
        while (allocResponse.getAllocatedContainers().size() < 2) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(2, allocated.size());
        for (Container container : allocated) {
            checkAllocatedContainer(container, 2 * GB, nm2.getNodeId(), 20);
        }
    } finally {
        if (rm != null) {
            rm.stop();
        }
    }
}

Example 55 with MockAM

use of org.apache.hadoop.yarn.server.resourcemanager.MockAM in project hadoop by apache.

the class TestSchedulingWithAllocationRequestId method testMultipleAppsWithAllocationReqId.

@Test
public void testMultipleAppsWithAllocationReqId() throws Exception {
    configureScheduler();
    YarnConfiguration conf = getConf();
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        // Register node1
        String host0 = "host_0";
        String host1 = "host_1";
        MockNM nm1 = new MockNM(host0 + ":1234", 8 * GB, rm.getResourceTrackerService());
        nm1.registerNode();
        // Register node2
        MockNM nm2 = new MockNM(host1 + ":2351", 8 * GB, rm.getResourceTrackerService());
        nm2.registerNode();
        // submit 1st app
        RMApp app1 = rm.submitApp(1 * GB, "user_0", "a1");
        MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
        // Submit app1 RR with allocationReqId = 5
        int numContainers = 1;
        am1.addRequests(new String[] { host0, host1 }, 1 * GB, 1, numContainers, 5L);
        AllocateResponse allocResponse = am1.schedule();
        // wait for containers to be allocated.
        nm1.nodeHeartbeat(true);
        // send the request
        allocResponse = am1.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        List<Container> allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 1 * GB, nm1.getNodeId(), 5L);
        // Submit another application
        RMApp app2 = rm.submitApp(1 * GB, "user_1", "a2");
        MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2);
        // Submit app2 RR with allocationReqId = 5
        am2.addRequests(new String[] { host0, host1 }, 2 * GB, 1, numContainers, 5L);
        am2.schedule();
        // wait for containers to be allocated.
        nm2.nodeHeartbeat(true);
        // send the request
        allocResponse = am2.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am2.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 2 * GB, nm2.getNodeId(), 5L);
        // Now submit app2 RR with allocationReqId = 10
        am2.addRequests(new String[] { host0, host1 }, 3 * GB, 1, numContainers, 10L);
        am2.schedule();
        // wait for containers to be allocated.
        nm1.nodeHeartbeat(true);
        // send the request
        allocResponse = am2.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am2.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 3 * GB, nm1.getNodeId(), 10L);
        // Now submit app1 RR with allocationReqId = 10
        am1.addRequests(new String[] { host0, host1 }, 4 * GB, 1, numContainers, 10L);
        am1.schedule();
        // wait for containers to be allocated.
        nm2.nodeHeartbeat(true);
        // send the request
        allocResponse = am1.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 4 * GB, nm2.getNodeId(), 10L);
    } finally {
        if (rm != null) {
            rm.stop();
        }
    }
}