Examples with RMApp org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp used on opensource projects

Example 56 with RMApp

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp in project hadoop by apache.

the class TestApplicationMasterService method testInvalidIncreaseDecreaseRequest.

@Test(timeout = 60000)
public void testInvalidIncreaseDecreaseRequest() throws Exception {
    conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class, ResourceScheduler.class);
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        // Register node1
        MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
        // Submit an application
        RMApp app1 = rm.submitApp(1024);
        // kick the scheduling
        nm1.nodeHeartbeat(true);
        RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
        MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
        RegisterApplicationMasterResponse registerResponse = am1.registerAppAttempt();
        sentRMContainerLaunched(rm, ContainerId.newContainerId(am1.getApplicationAttemptId(), 1));
        // Ask for a normal increase should be successfull
        am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(2048), null)));
        // Target resource is negative, should fail
        AllocateResponse response = am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(-1), null)));
        Assert.assertEquals(1, response.getUpdateErrors().size());
        Assert.assertEquals("RESOURCE_OUTSIDE_ALLOWED_RANGE", response.getUpdateErrors().get(0).getReason());
        // Target resource is more than maxAllocation, should fail
        response = am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.add(registerResponse.getMaximumResourceCapability(), Resources.createResource(1)), null)));
        Assert.assertEquals(1, response.getUpdateErrors().size());
        Assert.assertEquals("RESOURCE_OUTSIDE_ALLOWED_RANGE", response.getUpdateErrors().get(0).getReason());
        // Contains multiple increase/decrease requests for same contaienrId 
        response = am1.sendContainerResizingRequest(Arrays.asList(UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.INCREASE_RESOURCE, Resources.createResource(2048, 4), null), UpdateContainerRequest.newInstance(0, ContainerId.newContainerId(attempt1.getAppAttemptId(), 1), ContainerUpdateType.DECREASE_RESOURCE, Resources.createResource(1024, 1), null)));
        Assert.assertEquals(1, response.getUpdateErrors().size());
        Assert.assertEquals("UPDATE_OUTSTANDING_ERROR", response.getUpdateErrors().get(0).getReason());
    } finally {
        if (rm != null) {
            rm.close();
        }
    }
}

Example 57 with RMApp

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp in project hadoop by apache.

the class TestApplicationMasterService method testRMIdentifierOnContainerAllocation.

@Test(timeout = 3000000)
public void testRMIdentifierOnContainerAllocation() throws Exception {
    MockRM rm = new MockRM(conf);
    rm.start();
    // Register node1
    MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
    // Submit an application
    RMApp app1 = rm.submitApp(2048);
    // kick the scheduling
    nm1.nodeHeartbeat(true);
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
    am1.registerAppAttempt();
    am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1);
    // send the request
    AllocateResponse alloc1Response = am1.schedule();
    // kick the scheduler
    nm1.nodeHeartbeat(true);
    while (alloc1Response.getAllocatedContainers().size() < 1) {
        LOG.info("Waiting for containers to be created for app 1...");
        sleep(1000);
        alloc1Response = am1.schedule();
    }
    // assert RMIdentifer is set properly in allocated containers
    Container allocatedContainer = alloc1Response.getAllocatedContainers().get(0);
    ContainerTokenIdentifier tokenId = BuilderUtils.newContainerTokenIdentifier(allocatedContainer.getContainerToken());
    Assert.assertEquals(MockRM.getClusterTimeStamp(), tokenId.getRMIdentifier());
    rm.stop();
}

Example 58 with RMApp

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp in project hadoop by apache.

the class TestApplicationMasterService method testFinishApplicationMasterBeforeRegistering.

@Test(timeout = 1200000)
public void testFinishApplicationMasterBeforeRegistering() throws Exception {
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        // Register node1
        MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
        // Submit an application
        RMApp app1 = rm.submitApp(2048);
        MockAM am1 = MockRM.launchAM(app1, rm, nm1);
        FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.FAILED, "", "");
        try {
            am1.unregisterAppAttempt(req, false);
            Assert.fail("ApplicationMasterNotRegisteredException should be thrown");
        } catch (ApplicationMasterNotRegisteredException e) {
            Assert.assertNotNull(e);
            Assert.assertNotNull(e.getMessage());
            Assert.assertTrue(e.getMessage().contains("Application Master is trying to unregister before registering for:"));
        } catch (Exception e) {
            Assert.fail("ApplicationMasterNotRegisteredException should be thrown");
        }
        am1.registerAppAttempt();
        am1.unregisterAppAttempt(req, false);
        rm.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHING);
    } finally {
        if (rm != null) {
            rm.stop();
        }
    }
}

Example 59 with RMApp

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp in project hadoop by apache.

the class TestApplicationMasterService method testInvalidContainerReleaseRequest.

@Test(timeout = 600000)
public void testInvalidContainerReleaseRequest() throws Exception {
    MockRM rm = new MockRM(conf);
    try {
        rm.start();
        // Register node1
        MockNM nm1 = rm.registerNode("127.0.0.1:1234", 6 * GB);
        // Submit an application
        RMApp app1 = rm.submitApp(1024);
        // kick the scheduling
        nm1.nodeHeartbeat(true);
        RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
        MockAM am1 = rm.sendAMLaunched(attempt1.getAppAttemptId());
        am1.registerAppAttempt();
        am1.addRequests(new String[] { "127.0.0.1" }, GB, 1, 1);
        // send the request
        AllocateResponse alloc1Response = am1.schedule();
        // kick the scheduler
        nm1.nodeHeartbeat(true);
        while (alloc1Response.getAllocatedContainers().size() < 1) {
            LOG.info("Waiting for containers to be created for app 1...");
            sleep(1000);
            alloc1Response = am1.schedule();
        }
        Assert.assertTrue(alloc1Response.getAllocatedContainers().size() > 0);
        RMApp app2 = rm.submitApp(1024);
        nm1.nodeHeartbeat(true);
        RMAppAttempt attempt2 = app2.getCurrentAppAttempt();
        MockAM am2 = rm.sendAMLaunched(attempt2.getAppAttemptId());
        am2.registerAppAttempt();
        // Now trying to release container allocated for app1 -> appAttempt1.
        ContainerId cId = alloc1Response.getAllocatedContainers().get(0).getId();
        am2.addContainerToBeReleased(cId);
        try {
            am2.schedule();
            Assert.fail("Exception was expected!!");
        } catch (InvalidContainerReleaseException e) {
            StringBuilder sb = new StringBuilder("Cannot release container : ");
            sb.append(cId.toString());
            sb.append(" not belonging to this application attempt : ");
            sb.append(attempt2.getAppAttemptId().toString());
            Assert.assertTrue(e.getMessage().contains(sb.toString()));
        }
    } finally {
        if (rm != null) {
            rm.stop();
        }
    }
}

Example 60 with RMApp

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp in project hadoop by apache.

the class TestClientRMService method testAppSubmit.

@Test(timeout = 30000)
@SuppressWarnings("rawtypes")
public void testAppSubmit() throws Exception {
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration());
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext).getRMTimelineCollectorManager();
    ApplicationId appId1 = getApplicationId(100);
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);
    QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
    when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class), any(QueueACL.class), any(RMApp.class), any(String.class), any())).thenReturn(true);
    ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, appManager, mockAclsManager, mockQueueACLsManager, null);
    rmService.init(new Configuration());
    // without name and queue
    SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(appId1, null, null);
    try {
        rmService.submitApplication(submitRequest1);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    RMApp app1 = rmContext.getRMApps().get(appId1);
    Assert.assertNotNull("app doesn't exist", app1);
    Assert.assertEquals("app name doesn't match", YarnConfiguration.DEFAULT_APPLICATION_NAME, app1.getName());
    Assert.assertEquals("app queue doesn't match", YarnConfiguration.DEFAULT_QUEUE_NAME, app1.getQueue());
    // with name and queue
    String name = MockApps.newAppName();
    String queue = MockApps.newQueue();
    ApplicationId appId2 = getApplicationId(101);
    SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest(appId2, name, queue);
    submitRequest2.getApplicationSubmissionContext().setApplicationType("matchType");
    try {
        rmService.submitApplication(submitRequest2);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    RMApp app2 = rmContext.getRMApps().get(appId2);
    Assert.assertNotNull("app doesn't exist", app2);
    Assert.assertEquals("app name doesn't match", name, app2.getName());
    Assert.assertEquals("app queue doesn't match", queue, app2.getQueue());
    // duplicate appId
    try {
        rmService.submitApplication(submitRequest2);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    GetApplicationsRequest getAllAppsRequest = GetApplicationsRequest.newInstance(new HashSet<String>());
    GetApplicationsResponse getAllApplicationsResponse = rmService.getApplications(getAllAppsRequest);
    Assert.assertEquals(5, getAllApplicationsResponse.getApplicationList().size());
    Set<String> appTypes = new HashSet<String>();
    appTypes.add("matchType");
    getAllAppsRequest = GetApplicationsRequest.newInstance(appTypes);
    getAllApplicationsResponse = rmService.getApplications(getAllAppsRequest);
    Assert.assertEquals(1, getAllApplicationsResponse.getApplicationList().size());
    Assert.assertEquals(appId2, getAllApplicationsResponse.getApplicationList().get(0).getApplicationId());
}