Examples with YarnScheduler org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler used on opensource projects

Example 6 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestClientRMService method testAppSubmit.

@Test(timeout = 30000)
@SuppressWarnings("rawtypes")
public void testAppSubmit() throws Exception {
    YarnScheduler yarnScheduler = mockYarnScheduler();
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    RMStateStore stateStore = mock(RMStateStore.class);
    when(rmContext.getStateStore()).thenReturn(stateStore);
    RMAppManager appManager = new RMAppManager(rmContext, yarnScheduler, null, mock(ApplicationACLsManager.class), new Configuration());
    when(rmContext.getDispatcher().getEventHandler()).thenReturn(new EventHandler<Event>() {

        public void handle(Event event) {
        }
    });
    doReturn(mock(RMTimelineCollectorManager.class)).when(rmContext).getRMTimelineCollectorManager();
    ApplicationId appId1 = getApplicationId(100);
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);
    QueueACLsManager mockQueueACLsManager = mock(QueueACLsManager.class);
    when(mockQueueACLsManager.checkAccess(any(UserGroupInformation.class), any(QueueACL.class), any(RMApp.class), any(String.class), any())).thenReturn(true);
    ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, appManager, mockAclsManager, mockQueueACLsManager, null);
    rmService.init(new Configuration());
    // without name and queue
    SubmitApplicationRequest submitRequest1 = mockSubmitAppRequest(appId1, null, null);
    try {
        rmService.submitApplication(submitRequest1);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    RMApp app1 = rmContext.getRMApps().get(appId1);
    Assert.assertNotNull("app doesn't exist", app1);
    Assert.assertEquals("app name doesn't match", YarnConfiguration.DEFAULT_APPLICATION_NAME, app1.getName());
    Assert.assertEquals("app queue doesn't match", YarnConfiguration.DEFAULT_QUEUE_NAME, app1.getQueue());
    // with name and queue
    String name = MockApps.newAppName();
    String queue = MockApps.newQueue();
    ApplicationId appId2 = getApplicationId(101);
    SubmitApplicationRequest submitRequest2 = mockSubmitAppRequest(appId2, name, queue);
    submitRequest2.getApplicationSubmissionContext().setApplicationType("matchType");
    try {
        rmService.submitApplication(submitRequest2);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    RMApp app2 = rmContext.getRMApps().get(appId2);
    Assert.assertNotNull("app doesn't exist", app2);
    Assert.assertEquals("app name doesn't match", name, app2.getName());
    Assert.assertEquals("app queue doesn't match", queue, app2.getQueue());
    // duplicate appId
    try {
        rmService.submitApplication(submitRequest2);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    GetApplicationsRequest getAllAppsRequest = GetApplicationsRequest.newInstance(new HashSet<String>());
    GetApplicationsResponse getAllApplicationsResponse = rmService.getApplications(getAllAppsRequest);
    Assert.assertEquals(5, getAllApplicationsResponse.getApplicationList().size());
    Set<String> appTypes = new HashSet<String>();
    appTypes.add("matchType");
    getAllAppsRequest = GetApplicationsRequest.newInstance(appTypes);
    getAllApplicationsResponse = rmService.getApplications(getAllAppsRequest);
    Assert.assertEquals(1, getAllApplicationsResponse.getApplicationList().size());
    Assert.assertEquals(appId2, getAllApplicationsResponse.getApplicationList().get(0).getApplicationId());
}

Example 7 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestClientRMService method testGetApplicationReport.

@Test
public void testGetApplicationReport() throws Exception {
    YarnScheduler yarnScheduler = mock(YarnScheduler.class);
    RMContext rmContext = mock(RMContext.class);
    mockRMContext(yarnScheduler, rmContext);
    ApplicationId appId1 = getApplicationId(1);
    ApplicationACLsManager mockAclsManager = mock(ApplicationACLsManager.class);
    when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId1)).thenReturn(true);
    ClientRMService rmService = new ClientRMService(rmContext, yarnScheduler, null, mockAclsManager, null, null);
    try {
        RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
        GetApplicationReportRequest request = recordFactory.newRecordInstance(GetApplicationReportRequest.class);
        request.setApplicationId(appId1);
        GetApplicationReportResponse response = rmService.getApplicationReport(request);
        ApplicationReport report = response.getApplicationReport();
        ApplicationResourceUsageReport usageReport = report.getApplicationResourceUsageReport();
        Assert.assertEquals(10, usageReport.getMemorySeconds());
        Assert.assertEquals(3, usageReport.getVcoreSeconds());
        Assert.assertEquals("<Not set>", report.getAmNodeLabelExpression());
        Assert.assertEquals("<Not set>", report.getAppNodeLabelExpression());
        // if application has am node label set to blank
        ApplicationId appId2 = getApplicationId(2);
        when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId2)).thenReturn(true);
        request.setApplicationId(appId2);
        response = rmService.getApplicationReport(request);
        report = response.getApplicationReport();
        Assert.assertEquals(NodeLabel.DEFAULT_NODE_LABEL_PARTITION, report.getAmNodeLabelExpression());
        Assert.assertEquals(NodeLabel.NODE_LABEL_EXPRESSION_NOT_SET, report.getAppNodeLabelExpression());
        // if application has am node label set to blank
        ApplicationId appId3 = getApplicationId(3);
        when(mockAclsManager.checkAccess(UserGroupInformation.getCurrentUser(), ApplicationAccessType.VIEW_APP, null, appId3)).thenReturn(true);
        request.setApplicationId(appId3);
        response = rmService.getApplicationReport(request);
        report = response.getApplicationReport();
        Assert.assertEquals("high-mem", report.getAmNodeLabelExpression());
        Assert.assertEquals("high-mem", report.getAppNodeLabelExpression());
        // if application id is null
        GetApplicationReportRequest invalidRequest = recordFactory.newRecordInstance(GetApplicationReportRequest.class);
        invalidRequest.setApplicationId(null);
        try {
            rmService.getApplicationReport(invalidRequest);
        } catch (YarnException e) {
            // rmService should return a ApplicationNotFoundException
            // when a null application id is provided
            Assert.assertTrue(e instanceof ApplicationNotFoundException);
        }
    } finally {
        rmService.close();
    }
}

Example 8 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestWorkPreservingRMRestart method testAMfailedBetweenRMRestart.

// Test RM shuts down, in the meanwhile, AM fails. Restarted RM scheduler
// should not recover the containers that belong to the failed AM.
@Test(timeout = 20000)
public void testAMfailedBetweenRMRestart() throws Exception {
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS, 0);
    memStore.init(conf);
    rm1 = new MockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
    nm1.registerNode();
    RMApp app1 = rm1.submitApp(200);
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    rm2 = new MockRM(conf, memStore);
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    NMContainerStatus amContainer = TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    NMContainerStatus runningContainer = TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
    NMContainerStatus completedContainer = TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 3, ContainerState.COMPLETE);
    nm1.registerNode(Arrays.asList(amContainer, runningContainer, completedContainer), null);
    rm2.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
    // Wait for RM to settle down on recovering containers;
    Thread.sleep(3000);
    YarnScheduler scheduler = rm2.getResourceScheduler();
    // Previous AM failed, The failed AM should once again release the
    // just-recovered containers.
    assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
    assertNull(scheduler.getRMContainer(completedContainer.getContainerId()));
    rm2.waitForNewAMToLaunchAndRegister(app1.getApplicationId(), 2, nm1);
    MockNM nm2 = new MockNM("127.1.1.1:4321", 8192, rm2.getResourceTrackerService());
    NMContainerStatus previousAttemptContainer = TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 4, ContainerState.RUNNING);
    nm2.registerNode(Arrays.asList(previousAttemptContainer), null);
    // Wait for RM to settle down on recovering containers;
    Thread.sleep(3000);
    // check containers from previous failed attempt should not be recovered.
    assertNull(scheduler.getRMContainer(previousAttemptContainer.getContainerId()));
}

Example 9 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestRMAppAttemptTransitions method testScheduleTransitionReplaceAMContainerRequestWithDefaults.

@SuppressWarnings("unchecked")
@Test
public void testScheduleTransitionReplaceAMContainerRequestWithDefaults() {
    YarnScheduler mockScheduler = mock(YarnScheduler.class);
    when(mockScheduler.allocate(any(ApplicationAttemptId.class), any(List.class), any(List.class), any(List.class), any(List.class), any(ContainerUpdates.class))).thenAnswer(new Answer<Allocation>() {

        @SuppressWarnings("rawtypes")
        @Override
        public Allocation answer(InvocationOnMock invocation) throws Throwable {
            ResourceRequest rr = (ResourceRequest) ((List) invocation.getArguments()[1]).get(0);
            // capacity shouldn't changed
            assertEquals(Resource.newInstance(3333, 1), rr.getCapability());
            assertEquals("label-expression", rr.getNodeLabelExpression());
            // priority, #container, relax-locality will be changed
            assertEquals(RMAppAttemptImpl.AM_CONTAINER_PRIORITY, rr.getPriority());
            assertEquals(1, rr.getNumContainers());
            assertEquals(ResourceRequest.ANY, rr.getResourceName());
            // just return an empty allocation
            List l = new ArrayList();
            Set s = new HashSet();
            return new Allocation(l, Resources.none(), s, s, l);
        }
    });
    // create an attempt.
    applicationAttempt = new RMAppAttemptImpl(applicationAttempt.getAppAttemptId(), spyRMContext, scheduler, masterService, submissionContext, new Configuration(), ResourceRequest.newInstance(Priority.UNDEFINED, "host1", Resource.newInstance(3333, 1), 3, false, "label-expression"), application);
    new RMAppAttemptImpl.ScheduleTransition().transition((RMAppAttemptImpl) applicationAttempt, null);
}

Example 10 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestCapacityScheduler method testKillAllAppsInvalidSource.

@Test
public void testKillAllAppsInvalidSource() throws Exception {
    MockRM rm = setUpMove();
    YarnScheduler scheduler = rm.getResourceScheduler();
    // submit an app
    RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1");
    ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()).getCurrentApplicationAttemptId();
    // check preconditions
    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
    assertEquals(1, appsInA1.size());
    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
    assertTrue(appsInA.contains(appAttemptId));
    assertEquals(1, appsInA.size());
    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
    assertTrue(appsInRoot.contains(appAttemptId));
    assertEquals(1, appsInRoot.size());
    // now kill the app
    try {
        scheduler.killAllAppsInQueue("DOES_NOT_EXIST");
        Assert.fail();
    } catch (YarnException e) {
    // expected
    }
    // check postconditions, app should still be in a1
    appsInA1 = scheduler.getAppsInQueue("a1");
    assertEquals(1, appsInA1.size());
    appsInA = scheduler.getAppsInQueue("a");
    assertTrue(appsInA.contains(appAttemptId));
    assertEquals(1, appsInA.size());
    appsInRoot = scheduler.getAppsInQueue("root");
    assertTrue(appsInRoot.contains(appAttemptId));
    assertEquals(1, appsInRoot.size());
    rm.stop();
}