Examples with MockNM org.apache.hadoop.yarn.server.resourcemanager.MockNM used on opensource projects

Example 71 with MockNM

use of org.apache.hadoop.yarn.server.resourcemanager.MockNM in project hadoop by apache.

the class TestApplicationLimits method testApplicationLimitSubmit.

@Test(timeout = 120000)
public void testApplicationLimitSubmit() throws Exception {
    YarnConfiguration conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    // set node -> label
    mgr.addToCluserNodeLabelsWithDefaultExclusivity(ImmutableSet.of("x", "y", "z"));
    // set mapping:
    // h1 -> x
    // h2 -> y
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h1", 0), toSet("x")));
    mgr.addLabelsToNode(ImmutableMap.of(NodeId.newInstance("h2", 0), toSet("y")));
    // inject node label manager
    MockRM rm = new MockRM(getConfigurationWithQueueLabels(conf)) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.getRMContext().setNodeLabelManager(mgr);
    rm.start();
    MockNM nm1 = rm.registerNode("h1:1234", 4096);
    MockNM nm2 = rm.registerNode("h2:1234", 4096);
    MockNM nm3 = rm.registerNode("h3:1234", 4096);
    // Submit application to queue c where the default partition capacity is
    // zero
    RMApp app1 = rm.submitApp(GB, "app", "user", null, "c", false);
    rm.drainEvents();
    rm.waitForState(app1.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app1.getState());
    rm.killApp(app1.getApplicationId());
    RMApp app2 = rm.submitApp(GB, "app", "user", null, "a1", false);
    rm.drainEvents();
    rm.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app2.getState());
    // Check second application is rejected and based on queue level max
    // application app is rejected
    RMApp app3 = rm.submitApp(GB, "app", "user", null, "a1", false);
    rm.drainEvents();
    rm.waitForState(app3.getApplicationId(), RMAppState.FAILED);
    assertEquals(RMAppState.FAILED, app3.getState());
    assertEquals("org.apache.hadoop.security.AccessControlException: " + "Queue root.a.a1 already has 1 applications, cannot accept " + "submission of application: " + app3.getApplicationId(), app3.getDiagnostics().toString());
    // based on Global limit of queue usert application is rejected
    RMApp app11 = rm.submitApp(GB, "app", "user", null, "d", false);
    rm.drainEvents();
    rm.waitForState(app11.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app11.getState());
    RMApp app12 = rm.submitApp(GB, "app", "user", null, "d", false);
    rm.drainEvents();
    rm.waitForState(app12.getApplicationId(), RMAppState.ACCEPTED);
    assertEquals(RMAppState.ACCEPTED, app12.getState());
    RMApp app13 = rm.submitApp(GB, "app", "user", null, "d", false);
    rm.drainEvents();
    rm.waitForState(app13.getApplicationId(), RMAppState.FAILED);
    assertEquals(RMAppState.FAILED, app13.getState());
    assertEquals("org.apache.hadoop.security.AccessControlException: Queue" + " root.d already has 2 applications from user user cannot" + " accept submission of application: " + app13.getApplicationId(), app13.getDiagnostics().toString());
    // based on system max limit application is rejected
    RMApp app14 = rm.submitApp(GB, "app", "user2", null, "a2", false);
    rm.drainEvents();
    rm.waitForState(app14.getApplicationId(), RMAppState.ACCEPTED);
    RMApp app15 = rm.submitApp(GB, "app", "user2", null, "a2", false);
    rm.drainEvents();
    rm.waitForState(app15.getApplicationId(), RMAppState.FAILED);
    assertEquals(RMAppState.FAILED, app15.getState());
    assertEquals("Maximum system application limit reached,cannot" + " accept submission of application: " + app15.getApplicationId(), app15.getDiagnostics().toString());
    rm.killApp(app2.getApplicationId());
    rm.killApp(app11.getApplicationId());
    rm.killApp(app13.getApplicationId());
    rm.killApp(app14.getApplicationId());
    rm.stop();
}

Example 72 with MockNM

use of org.apache.hadoop.yarn.server.resourcemanager.MockNM in project hadoop by apache.

the class TestApplicationLimitsByPartition method testAtleastOneAMRunPerPartition.

@Test(timeout = 120000)
public void testAtleastOneAMRunPerPartition() throws Exception {
    /*
     * Test Case:
     * Even though am-resource-limit per queue/partition may cross if we
     * activate an app (high am resource demand), we have to activate it
     * since no other apps are running in that Queue/Partition. Here also
     * we run one test case for partition level and one in queue level to
     * ensure no breakage in existing functionality.
     *
     * Queue a1 supports labels (x,y). Configure am-resource-limit as 0.15 (x)
     * Queue c1 supports default label. Configure am-resource-limit as 0.15
     *
     * Queue A1 for label X can only support 1.5Gb AM resource.
     * Queue C1 (empty label) can support 1.5Gb AM resource.
     *
     * Verify atleast one AM is launched in each Queue.
     */
    simpleNodeLabelMappingToManager();
    CapacitySchedulerConfiguration config = (CapacitySchedulerConfiguration) TestUtils.getConfigurationWithQueueLabels(conf);
    // After getting queue conf, configure AM resource percent for Queue A1
    // as 0.15 (Label X) and for Queue C1 as 0.15 (Empty Label)
    final String A1 = CapacitySchedulerConfiguration.ROOT + ".a" + ".a1";
    final String C1 = CapacitySchedulerConfiguration.ROOT + ".c" + ".c1";
    config.setMaximumAMResourcePercentPerPartition(A1, "x", 0.15f);
    config.setMaximumApplicationMasterResourcePerQueuePercent(C1, 0.15f);
    // inject node label manager
    MockRM rm1 = new MockRM(config) {

        @Override
        public RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm1.getRMContext().setNodeLabelManager(mgr);
    rm1.start();
    // label = x
    MockNM nm1 = rm1.registerNode("h1:1234", 10 * GB);
    // label = y
    rm1.registerNode("h2:1234", 10 * GB);
    // label = <empty>
    MockNM nm3 = rm1.registerNode("h3:1234", 10 * GB);
    // Submit app1 (2 GB) to Queue A1 and label X
    RMApp app1 = rm1.submitApp(2 * GB, "app", "user", null, "a1", "x");
    // This app must be activated eventhough the am-resource per-partition
    // limit is only for 1.5GB.
    MockRM.launchAndRegisterAM(app1, rm1, nm1);
    // Submit 2nd app to label "X" with one GB and it must be pending since
    // am-resource per-partition limit is crossed (1.5 GB was the limit).
    rm1.submitApp(GB, "app", "user", null, "a1", "x");
    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
    LeafQueue leafQueue = (LeafQueue) cs.getQueue("a1");
    Assert.assertNotNull(leafQueue);
    // Only 1 app will be activated as am-limit for partition "x" is 0.15
    Assert.assertEquals(1, leafQueue.getNumActiveApplications());
    Assert.assertEquals(1, leafQueue.getNumPendingApplications());
    // Now verify the same test case in Queue C1 which takes default label
    // to see queue level am-resource-limit is still working as expected.
    // Submit an app to Queue C1 with empty label (2 GB)
    RMApp app3 = rm1.submitApp(2 * GB, "app", "user", null, "c1");
    // This app must be activated even though the am-resource per-queue
    // limit is only for 1.5GB
    MockRM.launchAndRegisterAM(app3, rm1, nm3);
    // Submit 2nd app to C1 (Default label, hence am-limit per-queue will be
    // considered).
    rm1.submitApp(GB, "app", "user", null, "c1");
    leafQueue = (LeafQueue) cs.getQueue("c1");
    Assert.assertNotNull(leafQueue);
    // 1 app will be activated (and it has AM resource more than queue limit)
    Assert.assertEquals(1, leafQueue.getNumActiveApplications());
    Assert.assertEquals(1, leafQueue.getNumPendingApplications());
    rm1.close();
}

Example 73 with MockNM

use of org.apache.hadoop.yarn.server.resourcemanager.MockNM in project hadoop by apache.

the class TestSchedulerUtils method testValidateResourceBlacklistRequest.

@Test
public void testValidateResourceBlacklistRequest() throws Exception {
    MyContainerManager containerManager = new MyContainerManager();
    final MockRMWithAMS rm = new MockRMWithAMS(new YarnConfiguration(), containerManager);
    rm.start();
    MockNM nm1 = rm.registerNode("localhost:1234", 5120);
    Map<ApplicationAccessType, String> acls = new HashMap<ApplicationAccessType, String>(2);
    acls.put(ApplicationAccessType.VIEW_APP, "*");
    RMApp app = rm.submitApp(1024, "appname", "appuser", acls);
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt = app.getCurrentAppAttempt();
    ApplicationAttemptId applicationAttemptId = attempt.getAppAttemptId();
    waitForLaunchedState(attempt);
    // Create a client to the RM.
    final Configuration conf = rm.getConfig();
    final YarnRPC rpc = YarnRPC.create(conf);
    UserGroupInformation currentUser = UserGroupInformation.createRemoteUser(applicationAttemptId.toString());
    Credentials credentials = containerManager.getContainerCredentials();
    final InetSocketAddress rmBindAddress = rm.getApplicationMasterService().getBindAddress();
    Token<? extends TokenIdentifier> amRMToken = MockRMWithAMS.setupAndReturnAMRMToken(rmBindAddress, credentials.getAllTokens());
    currentUser.addToken(amRMToken);
    ApplicationMasterProtocol client = currentUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {

        @Override
        public ApplicationMasterProtocol run() {
            return (ApplicationMasterProtocol) rpc.getProxy(ApplicationMasterProtocol.class, rmBindAddress, conf);
        }
    });
    RegisterApplicationMasterRequest request = Records.newRecord(RegisterApplicationMasterRequest.class);
    client.registerApplicationMaster(request);
    ResourceBlacklistRequest blacklistRequest = ResourceBlacklistRequest.newInstance(Collections.singletonList(ResourceRequest.ANY), null);
    AllocateRequest allocateRequest = AllocateRequest.newInstance(0, 0.0f, null, null, blacklistRequest);
    boolean error = false;
    try {
        client.allocate(allocateRequest);
    } catch (InvalidResourceBlacklistRequestException e) {
        error = true;
    }
    rm.stop();
    Assert.assertTrue("Didn't not catch InvalidResourceBlacklistRequestException", error);
}

Example 74 with MockNM

use of org.apache.hadoop.yarn.server.resourcemanager.MockNM in project hadoop by apache.

the class TestSchedulingWithAllocationRequestId method testMultipleAllocationRequestIds.

@Test
public void testMultipleAllocationRequestIds() throws Exception {
    configureScheduler();
    YarnConfiguration conf = getConf();
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        MockNM nm1 = rm.registerNode("127.0.0.1:1234", 4 * GB);
        MockNM nm2 = rm.registerNode("127.0.0.2:5678", 4 * GB);
        RMApp app1 = rm.submitApp(2048);
        // kick the scheduling
        nm1.nodeHeartbeat(true);
        RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
        MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
        am1.registerAppAttempt();
        // add request for containers with id 10 & 20
        am1.addRequests(new String[] { "127.0.0.1" }, 2 * GB, 1, 1, 10L);
        // send the request
        AllocateResponse allocResponse = am1.schedule();
        am1.addRequests(new String[] { "127.0.0.2" }, 2 * GB, 1, 2, 20L);
        // send the request
        allocResponse = am1.schedule();
        // check if request id 10 is satisfied
        nm1.nodeHeartbeat(true);
        // send the request
        allocResponse = am1.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        List<Container> allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 2 * GB, nm1.getNodeId(), 10);
        // check now if request id 20 is satisfied
        nm2.nodeHeartbeat(true);
        while (allocResponse.getAllocatedContainers().size() < 2) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(2, allocated.size());
        for (Container container : allocated) {
            checkAllocatedContainer(container, 2 * GB, nm2.getNodeId(), 20);
        }
    } finally {
        if (rm != null) {
            rm.stop();
        }
    }
}

Example 75 with MockNM

use of org.apache.hadoop.yarn.server.resourcemanager.MockNM in project hadoop by apache.

the class TestSchedulingWithAllocationRequestId method testMultipleAppsWithAllocationReqId.

@Test
public void testMultipleAppsWithAllocationReqId() throws Exception {
    configureScheduler();
    YarnConfiguration conf = getConf();
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        // Register node1
        String host0 = "host_0";
        String host1 = "host_1";
        MockNM nm1 = new MockNM(host0 + ":1234", 8 * GB, rm.getResourceTrackerService());
        nm1.registerNode();
        // Register node2
        MockNM nm2 = new MockNM(host1 + ":2351", 8 * GB, rm.getResourceTrackerService());
        nm2.registerNode();
        // submit 1st app
        RMApp app1 = rm.submitApp(1 * GB, "user_0", "a1");
        MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
        // Submit app1 RR with allocationReqId = 5
        int numContainers = 1;
        am1.addRequests(new String[] { host0, host1 }, 1 * GB, 1, numContainers, 5L);
        AllocateResponse allocResponse = am1.schedule();
        // wait for containers to be allocated.
        nm1.nodeHeartbeat(true);
        // send the request
        allocResponse = am1.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        List<Container> allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 1 * GB, nm1.getNodeId(), 5L);
        // Submit another application
        RMApp app2 = rm.submitApp(1 * GB, "user_1", "a2");
        MockAM am2 = MockRM.launchAndRegisterAM(app2, rm, nm2);
        // Submit app2 RR with allocationReqId = 5
        am2.addRequests(new String[] { host0, host1 }, 2 * GB, 1, numContainers, 5L);
        am2.schedule();
        // wait for containers to be allocated.
        nm2.nodeHeartbeat(true);
        // send the request
        allocResponse = am2.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am2.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 2 * GB, nm2.getNodeId(), 5L);
        // Now submit app2 RR with allocationReqId = 10
        am2.addRequests(new String[] { host0, host1 }, 3 * GB, 1, numContainers, 10L);
        am2.schedule();
        // wait for containers to be allocated.
        nm1.nodeHeartbeat(true);
        // send the request
        allocResponse = am2.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am2.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 3 * GB, nm1.getNodeId(), 10L);
        // Now submit app1 RR with allocationReqId = 10
        am1.addRequests(new String[] { host0, host1 }, 4 * GB, 1, numContainers, 10L);
        am1.schedule();
        // wait for containers to be allocated.
        nm2.nodeHeartbeat(true);
        // send the request
        allocResponse = am1.schedule();
        while (allocResponse.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            Thread.sleep(100);
            allocResponse = am1.schedule();
        }
        allocated = allocResponse.getAllocatedContainers();
        Assert.assertEquals(1, allocated.size());
        checkAllocatedContainer(allocated.get(0), 4 * GB, nm2.getNodeId(), 10L);
    } finally {
        if (rm != null) {
            rm.stop();
        }
    }
}