Examples with RMApp org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp used on opensource projects

Example 96 with RMApp

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp in project hadoop by apache.

the class TestRM method testApplicationKillAtAcceptedState.

/**
   * Validate killing an application when it is at accepted state.
   * @throws Exception exception
   */
@Test(timeout = 60000)
public void testApplicationKillAtAcceptedState() throws Exception {
    final Dispatcher dispatcher = new DrainDispatcher() {

        @Override
        public EventHandler<Event> getEventHandler() {
            class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {

                @Override
                public boolean matches(Object argument) {
                    if (argument instanceof RMAppAttemptEvent) {
                        if (((RMAppAttemptEvent) argument).getType().equals(RMAppAttemptEventType.KILL)) {
                            return true;
                        }
                    }
                    return false;
                }
            }
            EventHandler handler = spy(super.getEventHandler());
            doNothing().when(handler).handle(argThat(new EventArgMatcher()));
            return handler;
        }
    };
    MockRM rm = new MockRM(conf) {

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }
    };
    // test metrics
    QueueMetrics metrics = rm.getResourceScheduler().getRootQueueMetrics();
    int appsKilled = metrics.getAppsKilled();
    int appsSubmitted = metrics.getAppsSubmitted();
    rm.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm.getResourceTrackerService());
    nm1.registerNode();
    // a failed app
    RMApp application = rm.submitApp(200);
    MockAM am = MockRM.launchAM(application, rm, nm1);
    rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.LAUNCHED);
    nm1.nodeHeartbeat(am.getApplicationAttemptId(), 1, ContainerState.RUNNING);
    rm.waitForState(application.getApplicationId(), RMAppState.ACCEPTED);
    // Now kill the application before new attempt is launched, the app report
    // returns the invalid AM host and port.
    KillApplicationRequest request = KillApplicationRequest.newInstance(application.getApplicationId());
    rm.getClientRMService().forceKillApplication(request);
    // Specific test for YARN-1689 follows
    // Now let's say a race causes AM to register now. This should not crash RM.
    am.registerAppAttempt(false);
    // We explicitly intercepted the kill-event to RMAppAttempt, so app should
    // still be in KILLING state.
    rm.waitForState(application.getApplicationId(), RMAppState.KILLING);
    // AM should now be in running
    rm.waitForState(am.getApplicationAttemptId(), RMAppAttemptState.RUNNING);
    // Simulate that appAttempt is killed.
    rm.getRMContext().getDispatcher().getEventHandler().handle(new RMAppEvent(application.getApplicationId(), RMAppEventType.ATTEMPT_KILLED));
    rm.waitForState(application.getApplicationId(), RMAppState.KILLED);
    // test metrics
    metrics = rm.getResourceScheduler().getRootQueueMetrics();
    Assert.assertEquals(appsKilled + 1, metrics.getAppsKilled());
    Assert.assertEquals(appsSubmitted + 1, metrics.getAppsSubmitted());
}

Example 97 with RMApp

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp in project hadoop by apache.

the class TestRM method testKillFinishingApp.

// Test Kill an app while the app is finishing in the meanwhile.
@Test(timeout = 30000)
public void testKillFinishingApp() throws Exception {
    // this dispatcher ignores RMAppAttemptEventType.KILL event
    final Dispatcher dispatcher = new DrainDispatcher() {

        @Override
        public EventHandler<Event> getEventHandler() {
            class EventArgMatcher extends ArgumentMatcher<AbstractEvent> {

                @Override
                public boolean matches(Object argument) {
                    if (argument instanceof RMAppAttemptEvent) {
                        if (((RMAppAttemptEvent) argument).getType().equals(RMAppAttemptEventType.KILL)) {
                            return true;
                        }
                    }
                    return false;
                }
            }
            EventHandler handler = spy(super.getEventHandler());
            doNothing().when(handler).handle(argThat(new EventArgMatcher()));
            return handler;
        }
    };
    MockRM rm1 = new MockRM(conf) {

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }
    };
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
    nm1.registerNode();
    RMApp app1 = rm1.submitApp(200);
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    rm1.killApp(app1.getApplicationId());
    FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "", "");
    am1.unregisterAppAttempt(req, true);
    rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHING);
    nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FINISHED);
    rm1.waitForState(app1.getApplicationId(), RMAppState.FINISHED);
}

Example 98 with RMApp

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp in project hadoop by apache.

the class TestRM method testInvalidatedAMHostPortOnAMRestart.

@Test(timeout = 60000)
public void testInvalidatedAMHostPortOnAMRestart() throws Exception {
    MockRM rm1 = new MockRM(conf);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // a failed app
    RMApp app2 = rm1.submitApp(200);
    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
    nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.FAILED);
    rm1.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED);
    // before new attempt is launched, the app report returns the invalid AM
    // host and port.
    GetApplicationReportRequest request1 = GetApplicationReportRequest.newInstance(app2.getApplicationId());
    ApplicationReport report1 = rm1.getClientRMService().getApplicationReport(request1).getApplicationReport();
    Assert.assertEquals("N/A", report1.getHost());
    Assert.assertEquals(-1, report1.getRpcPort());
}

Example 99 with RMApp

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp in project hadoop by apache.

the class TestRM method testInvalidateAMHostPortWhenAMFailedOrKilled.

// This is to test AM Host and rpc port are invalidated after the am attempt
// is killed or failed, so that client doesn't get the wrong information.
@Test(timeout = 80000)
public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
    MockRM rm1 = new MockRM(conf);
    rm1.start();
    // a succeeded app
    RMApp app1 = rm1.submitApp(200);
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);
    // a failed app
    RMApp app2 = rm1.submitApp(200);
    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
    nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.FAILED);
    rm1.waitForState(app2.getApplicationId(), RMAppState.FAILED);
    // a killed app
    RMApp app3 = rm1.submitApp(200);
    MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1);
    rm1.killApp(app3.getApplicationId());
    rm1.waitForState(app3.getApplicationId(), RMAppState.KILLED);
    rm1.waitForState(am3.getApplicationAttemptId(), RMAppAttemptState.KILLED);
    GetApplicationsRequest request1 = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.FINISHED, YarnApplicationState.KILLED, YarnApplicationState.FAILED));
    GetApplicationsResponse response1 = rm1.getClientRMService().getApplications(request1);
    List<ApplicationReport> appList1 = response1.getApplicationList();
    Assert.assertEquals(3, appList1.size());
    for (ApplicationReport report : appList1) {
        // killed/failed apps host and rpc port are invalidated.
        if (report.getApplicationId().equals(app2.getApplicationId()) || report.getApplicationId().equals(app3.getApplicationId())) {
            Assert.assertEquals("N/A", report.getHost());
            Assert.assertEquals(-1, report.getRpcPort());
        }
        // succeeded app's host and rpc port is not invalidated
        if (report.getApplicationId().equals(app1.getApplicationId())) {
            Assert.assertFalse(report.getHost().equals("N/A"));
            Assert.assertTrue(report.getRpcPort() != -1);
        }
    }
}

Example 100 with RMApp

use of org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp in project hadoop by apache.

the class TestRM method testActivatingApplicationAfterAddingNM.

@Test(timeout = 300000)
public void testActivatingApplicationAfterAddingNM() throws Exception {
    MockRM rm1 = new MockRM(conf);
    // start like normal because state is empty
    rm1.start();
    // app that gets launched
    RMApp app1 = rm1.submitApp(200);
    // app that does not get launched
    RMApp app2 = rm1.submitApp(200);
    // app1 and app2 should be scheduled, but because no resource is available,
    // they are not activated.
    RMAppAttempt attempt1 = app1.getCurrentAppAttempt();
    ApplicationAttemptId attemptId1 = attempt1.getAppAttemptId();
    rm1.waitForState(attemptId1, RMAppAttemptState.SCHEDULED);
    RMAppAttempt attempt2 = app2.getCurrentAppAttempt();
    ApplicationAttemptId attemptId2 = attempt2.getAppAttemptId();
    rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED);
    MockNM nm1 = new MockNM("h1:1234", 15120, rm1.getResourceTrackerService());
    MockNM nm2 = new MockNM("h2:5678", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    nm2.registerNode();
    //kick the scheduling
    nm1.nodeHeartbeat(true);
    // app1 should be allocated now
    rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
    rm1.waitForState(attemptId2, RMAppAttemptState.SCHEDULED);
    nm2.nodeHeartbeat(true);
    // app2 should be allocated now
    rm1.waitForState(attemptId1, RMAppAttemptState.ALLOCATED);
    rm1.waitForState(attemptId2, RMAppAttemptState.ALLOCATED);
    rm1.stop();
}