Examples with YarnScheduler org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler used on opensource projects

Example 16 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestRMRestart method testAppRecoveredInOrderOnRMRestart.

@Test(timeout = 20000)
public void testAppRecoveredInOrderOnRMRestart() throws Exception {
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    for (int i = 10; i > 0; i--) {
        ApplicationStateData appState = mock(ApplicationStateData.class);
        ApplicationSubmissionContext context = mock(ApplicationSubmissionContext.class);
        when(appState.getApplicationSubmissionContext()).thenReturn(context);
        when(context.getApplicationId()).thenReturn(ApplicationId.newInstance(1234, i));
        memStore.getState().getApplicationState().put(appState.getApplicationSubmissionContext().getApplicationId(), appState);
    }
    MockRM rm1 = new MockRM(conf, memStore) {

        @Override
        protected RMAppManager createRMAppManager() {
            return new TestRMAppManager(this.rmContext, this.scheduler, this.masterService, this.applicationACLsManager, conf);
        }

        class TestRMAppManager extends RMAppManager {

            ApplicationId prevId = ApplicationId.newInstance(1234, 0);

            public TestRMAppManager(RMContext context, YarnScheduler scheduler, ApplicationMasterService masterService, ApplicationACLsManager applicationACLsManager, Configuration conf) {
                super(context, scheduler, masterService, applicationACLsManager, conf);
            }

            @Override
            protected void recoverApplication(ApplicationStateData appState, RMState rmState) throws Exception {
                // check application is recovered in order.
                Assert.assertTrue(rmState.getApplicationState().size() > 0);
                Assert.assertTrue(appState.getApplicationSubmissionContext().getApplicationId().compareTo(prevId) > 0);
                prevId = appState.getApplicationSubmissionContext().getApplicationId();
            }
        }
    };
    try {
        rm1.start();
    } finally {
        rm1.stop();
    }
}

Example 17 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestCapacityScheduler method testMoveAllAppsInvalidDestination.

@Test
public void testMoveAllAppsInvalidDestination() throws Exception {
    MockRM rm = setUpMove();
    YarnScheduler scheduler = rm.getResourceScheduler();
    // submit an app
    RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1");
    ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()).getCurrentApplicationAttemptId();
    // check preconditions
    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
    assertEquals(1, appsInA1.size());
    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
    assertTrue(appsInA.contains(appAttemptId));
    assertEquals(1, appsInA.size());
    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
    assertTrue(appsInRoot.contains(appAttemptId));
    assertEquals(1, appsInRoot.size());
    List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
    assertTrue(appsInB1.isEmpty());
    List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b");
    assertTrue(appsInB.isEmpty());
    // now move the app
    try {
        scheduler.moveAllApps("a1", "DOES_NOT_EXIST");
        Assert.fail();
    } catch (YarnException e) {
    // expected
    }
    // check postconditions, app should still be in a1
    appsInA1 = scheduler.getAppsInQueue("a1");
    assertEquals(1, appsInA1.size());
    appsInA = scheduler.getAppsInQueue("a");
    assertTrue(appsInA.contains(appAttemptId));
    assertEquals(1, appsInA.size());
    appsInRoot = scheduler.getAppsInQueue("root");
    assertTrue(appsInRoot.contains(appAttemptId));
    assertEquals(1, appsInRoot.size());
    appsInB1 = scheduler.getAppsInQueue("b1");
    assertTrue(appsInB1.isEmpty());
    appsInB = scheduler.getAppsInQueue("b");
    assertTrue(appsInB.isEmpty());
    rm.stop();
}

Example 18 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestCapacityScheduler method testMoveAllAppsInvalidSource.

@Test
public void testMoveAllAppsInvalidSource() throws Exception {
    MockRM rm = setUpMove();
    YarnScheduler scheduler = rm.getResourceScheduler();
    // submit an app
    RMApp app = rm.submitApp(GB, "test-move-1", "user_0", null, "a1");
    ApplicationAttemptId appAttemptId = rm.getApplicationReport(app.getApplicationId()).getCurrentApplicationAttemptId();
    // check preconditions
    List<ApplicationAttemptId> appsInA1 = scheduler.getAppsInQueue("a1");
    assertEquals(1, appsInA1.size());
    List<ApplicationAttemptId> appsInA = scheduler.getAppsInQueue("a");
    assertTrue(appsInA.contains(appAttemptId));
    assertEquals(1, appsInA.size());
    List<ApplicationAttemptId> appsInRoot = scheduler.getAppsInQueue("root");
    assertTrue(appsInRoot.contains(appAttemptId));
    assertEquals(1, appsInRoot.size());
    List<ApplicationAttemptId> appsInB1 = scheduler.getAppsInQueue("b1");
    assertTrue(appsInB1.isEmpty());
    List<ApplicationAttemptId> appsInB = scheduler.getAppsInQueue("b");
    assertTrue(appsInB.isEmpty());
    // now move the app
    try {
        scheduler.moveAllApps("DOES_NOT_EXIST", "b1");
        Assert.fail();
    } catch (YarnException e) {
    // expected
    }
    // check postconditions, app should still be in a1
    appsInA1 = scheduler.getAppsInQueue("a1");
    assertEquals(1, appsInA1.size());
    appsInA = scheduler.getAppsInQueue("a");
    assertTrue(appsInA.contains(appAttemptId));
    assertEquals(1, appsInA.size());
    appsInRoot = scheduler.getAppsInQueue("root");
    assertTrue(appsInRoot.contains(appAttemptId));
    assertEquals(1, appsInRoot.size());
    appsInB1 = scheduler.getAppsInQueue("b1");
    assertTrue(appsInB1.isEmpty());
    appsInB = scheduler.getAppsInQueue("b");
    assertTrue(appsInB.isEmpty());
    rm.stop();
}

Example 19 with YarnScheduler

use of org.apache.hadoop.yarn.server.resourcemanager.scheduler.YarnScheduler in project hadoop by apache.

the class TestWorkPreservingRMRestart method testContainersNotRecoveredForCompletedApps.

// Apps already completed before RM restart. Restarted RM scheduler should not
// recover containers for completed apps.
@Test(timeout = 20000)
public void testContainersNotRecoveredForCompletedApps() throws Exception {
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    rm1 = new MockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
    nm1.registerNode();
    RMApp app1 = rm1.submitApp(200);
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);
    rm2 = new MockRM(conf, memStore);
    rm2.start();
    nm1.setResourceTrackerService(rm2.getResourceTrackerService());
    NMContainerStatus runningContainer = TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 2, ContainerState.RUNNING);
    NMContainerStatus completedContainer = TestRMRestart.createNMContainerStatus(am1.getApplicationAttemptId(), 3, ContainerState.COMPLETE);
    nm1.registerNode(Arrays.asList(runningContainer, completedContainer), null);
    RMApp recoveredApp1 = rm2.getRMContext().getRMApps().get(app1.getApplicationId());
    assertEquals(RMAppState.FINISHED, recoveredApp1.getState());
    // Wait for RM to settle down on recovering containers;
    Thread.sleep(3000);
    YarnScheduler scheduler = rm2.getResourceScheduler();
    // scheduler should not recover containers for finished apps.
    assertNull(scheduler.getRMContainer(runningContainer.getContainerId()));
    assertNull(scheduler.getRMContainer(completedContainer.getContainerId()));
}