Examples with MockNM org.apache.hadoop.yarn.server.resourcemanager.MockNM used on opensource projects

Example 81 with MockNM

use of org.apache.hadoop.yarn.server.resourcemanager.MockNM in project hadoop by apache.

the class TestApplicationPriority method testUpdatePriorityOnPendingAppAndKillAttempt.

@Test(timeout = 120000)
public void testUpdatePriorityOnPendingAppAndKillAttempt() throws Exception {
    int maxPriority = 10;
    int appPriority = 5;
    YarnConfiguration conf = new YarnConfiguration();
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, maxPriority);
    MockRM rm = new MockRM(conf);
    rm.init(conf);
    rm.start();
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    CSQueue defaultQueue = (LeafQueue) cs.getQueue("default");
    // Update priority and kill application with no resource
    RMApp app1 = rm.submitApp(1024, Priority.newInstance(appPriority));
    Collection<FiCaSchedulerApp> appsPending = ((LeafQueue) defaultQueue).getPendingApplications();
    Collection<FiCaSchedulerApp> activeApps = ((LeafQueue) defaultQueue).getOrderingPolicy().getSchedulableEntities();
    // Verify app is in pending state
    Assert.assertEquals("Pending apps should be 1", 1, appsPending.size());
    Assert.assertEquals("Active apps should be 0", 0, activeApps.size());
    // kill app1 which is pending
    killAppAndVerifyOrderingPolicy(rm, defaultQueue, 0, 0, app1);
    // Check ordering policy size when resource is added
    MockNM nm1 = new MockNM("127.0.0.1:1234", 8096, rm.getResourceTrackerService());
    nm1.registerNode();
    RMApp app2 = rm.submitApp(1024, Priority.newInstance(appPriority));
    Assert.assertEquals("Pending apps should be 0", 0, appsPending.size());
    Assert.assertEquals("Active apps should be 1", 1, activeApps.size());
    RMApp app3 = rm.submitApp(1024, Priority.newInstance(appPriority));
    RMApp app4 = rm.submitApp(1024, Priority.newInstance(appPriority));
    Assert.assertEquals("Pending apps should be 2", 2, appsPending.size());
    Assert.assertEquals("Active apps should be 1", 1, activeApps.size());
    // kill app3, pending apps should reduce to 1
    killAppAndVerifyOrderingPolicy(rm, defaultQueue, 1, 1, app3);
    // kill app2, running apps is killed and pending added to running
    killAppAndVerifyOrderingPolicy(rm, defaultQueue, 0, 1, app2);
    // kill app4, all apps are killed and both policy size should be zero
    killAppAndVerifyOrderingPolicy(rm, defaultQueue, 0, 0, app4);
    rm.stop();
}

Example 82 with MockNM

use of org.apache.hadoop.yarn.server.resourcemanager.MockNM in project hadoop by apache.

the class TestApplicationPriority method testUpdatePriorityAtRuntime.

@Test
public void testUpdatePriorityAtRuntime() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    // Set Max Application Priority as 10
    conf.setInt(YarnConfiguration.MAX_CLUSTER_LEVEL_APPLICATION_PRIORITY, 10);
    MockRM rm = new MockRM(conf);
    rm.start();
    Priority appPriority1 = Priority.newInstance(5);
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 16 * GB);
    RMApp app1 = rm.submitApp(1 * GB, appPriority1);
    // kick the scheduler, 1 GB given to AM1, remaining 15GB on nm1
    MockAM am1 = MockRM.launchAM(app1, rm, nm1);
    am1.registerAppAttempt();
    // get scheduler
    CapacityScheduler cs = (CapacityScheduler) rm.getResourceScheduler();
    // Change the priority of App1 to 8
    Priority appPriority2 = Priority.newInstance(8);
    UserGroupInformation ugi = UserGroupInformation.createRemoteUser(app1.getUser());
    cs.updateApplicationPriority(appPriority2, app1.getApplicationId(), null, ugi);
    // get scheduler app
    FiCaSchedulerApp schedulerAppAttempt = cs.getSchedulerApplications().get(app1.getApplicationId()).getCurrentAppAttempt();
    // Verify whether the new priority is updated
    Assert.assertEquals(appPriority2, schedulerAppAttempt.getPriority());
}

Example 83 with MockNM

use of org.apache.hadoop.yarn.server.resourcemanager.MockNM in project hadoop by apache.

the class TestCapacityScheduler method testRecoverRequestAfterPreemption.

@Test(timeout = 300000)
public void testRecoverRequestAfterPreemption() throws Exception {
    Configuration conf = new Configuration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm1 = new MockRM(conf);
    rm1.start();
    MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 8000);
    RMApp app1 = rm1.submitApp(1024);
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    CapacityScheduler cs = (CapacityScheduler) rm1.getResourceScheduler();
    // request a container.
    am1.allocate("127.0.0.1", 1024, 1, new ArrayList<ContainerId>());
    ContainerId containerId1 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    rm1.waitForState(nm1, containerId1, RMContainerState.ALLOCATED);
    RMContainer rmContainer = cs.getRMContainer(containerId1);
    List<ResourceRequest> requests = rmContainer.getResourceRequests();
    FiCaSchedulerApp app = cs.getApplicationAttempt(am1.getApplicationAttemptId());
    FiCaSchedulerNode node = cs.getNode(rmContainer.getAllocatedNode());
    for (ResourceRequest request : requests) {
        // Skip the OffRack and RackLocal resource requests.
        if (request.getResourceName().equals(node.getRackName()) || request.getResourceName().equals(ResourceRequest.ANY)) {
            continue;
        }
        // Already the node local resource request is cleared from RM after
        // allocation.
        Assert.assertEquals(0, app.getOutstandingAsksCount(SchedulerRequestKey.create(request), request.getResourceName()));
    }
    // Call killContainer to preempt the container
    cs.markContainerForKillable(rmContainer);
    Assert.assertEquals(3, requests.size());
    for (ResourceRequest request : requests) {
        // Resource request must have added back in RM after preempt event
        // handling.
        Assert.assertEquals(1, app.getOutstandingAsksCount(SchedulerRequestKey.create(request), request.getResourceName()));
    }
    // New container will be allocated and will move to ALLOCATED state
    ContainerId containerId2 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
    rm1.waitForState(nm1, containerId2, RMContainerState.ALLOCATED);
    // allocate container
    List<Container> containers = am1.allocate(new ArrayList<ResourceRequest>(), new ArrayList<ContainerId>()).getAllocatedContainers();
    // Now with updated ResourceRequest, a container is allocated for AM.
    Assert.assertTrue(containers.size() == 1);
}

Example 84 with MockNM

use of org.apache.hadoop.yarn.server.resourcemanager.MockNM in project hadoop by apache.

the class TestCapacityScheduler method testPendingResourceUpdatedAccordingToIncreaseRequestChanges.

@Test
public void testPendingResourceUpdatedAccordingToIncreaseRequestChanges() throws Exception {
    Configuration conf = TestUtils.getConfigurationWithQueueLabels(new Configuration(false));
    conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
    final RMNodeLabelsManager mgr = new NullRMNodeLabelsManager();
    mgr.init(conf);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    MockRM rm = new MockRM(conf, memStore) {

        protected RMNodeLabelsManager createNodeLabelManager() {
            return mgr;
        }
    };
    rm.start();
    // label = ""
    MockNM nm1 = new MockNM("h1:1234", 200 * GB, rm.getResourceTrackerService());
    nm1.registerNode();
    // Launch app1 in queue=a1
    RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "a1");
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
    // Allocate two more containers
    am1.allocate(Arrays.asList(ResourceRequest.newInstance(Priority.newInstance(1), "*", Resources.createResource(2 * GB), 2)), null);
    ContainerId containerId1 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 1);
    ContainerId containerId2 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
    ContainerId containerId3 = ContainerId.newContainerId(am1.getApplicationAttemptId(), 3);
    Assert.assertTrue(rm.waitForState(nm1, containerId3, RMContainerState.ALLOCATED));
    // Acquire them
    am1.allocate(null, null);
    sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 1L));
    sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 2L));
    sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 3L));
    // am1 asks to change its AM container from 1GB to 3GB
    am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, containerId1, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(3 * GB), null)));
    FiCaSchedulerApp app = getFiCaSchedulerApp(rm, app1.getApplicationId());
    Assert.assertEquals(2 * GB, app.getAppAttemptResourceUsage().getPending().getMemorySize());
    checkPendingResource(rm, "a1", 2 * GB, null);
    checkPendingResource(rm, "a", 2 * GB, null);
    checkPendingResource(rm, "root", 2 * GB, null);
    // am1 asks to change containerId2 (2G -> 3G) and containerId3 (2G -> 5G)
    am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, containerId2, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(3 * GB), null), UpdateContainerRequest.newInstance(0, containerId3, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(5 * GB), null)));
    Assert.assertEquals(6 * GB, app.getAppAttemptResourceUsage().getPending().getMemorySize());
    checkPendingResource(rm, "a1", 6 * GB, null);
    checkPendingResource(rm, "a", 6 * GB, null);
    checkPendingResource(rm, "root", 6 * GB, null);
    // am1 asks to change containerId1 (1G->3G), containerId2 (2G -> 4G) and
    // containerId3 (2G -> 2G)
    am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, containerId1, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(3 * GB), null), UpdateContainerRequest.newInstance(0, containerId2, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(4 * GB), null), UpdateContainerRequest.newInstance(0, containerId3, ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(2 * GB), null)));
    Assert.assertEquals(4 * GB, app.getAppAttemptResourceUsage().getPending().getMemorySize());
    checkPendingResource(rm, "a1", 4 * GB, null);
    checkPendingResource(rm, "a", 4 * GB, null);
    checkPendingResource(rm, "root", 4 * GB, null);
}

Example 85 with MockNM

use of org.apache.hadoop.yarn.server.resourcemanager.MockNM in project hadoop by apache.

the class TestRMWebServicesAppsModification method testGetAppQueue.

@Test
public void testGetAppQueue() throws Exception {
    client().addFilter(new LoggingFilter(System.out));
    boolean isCapacityScheduler = rm.getResourceScheduler() instanceof CapacityScheduler;
    rm.start();
    MockNM amNodeManager = rm.registerNode("127.0.0.1:1234", 2048);
    String[] contentTypes = { MediaType.APPLICATION_JSON, MediaType.APPLICATION_XML };
    for (String contentType : contentTypes) {
        RMApp app = rm.submitApp(CONTAINER_MB, "", webserviceUserName);
        amNodeManager.nodeHeartbeat(true);
        ClientResponse response = this.constructWebResource("apps", app.getApplicationId().toString(), "queue").accept(contentType).get(ClientResponse.class);
        assertResponseStatusCode(Status.OK, response.getStatusInfo());
        String expectedQueue = "default";
        if (!isCapacityScheduler) {
            expectedQueue = "root." + webserviceUserName;
        }
        if (contentType.contains(MediaType.APPLICATION_JSON)) {
            verifyAppQueueJson(response, expectedQueue);
        } else {
            verifyAppQueueXML(response, expectedQueue);
        }
    }
    rm.stop();
}