Examples with ApplicationReport org.apache.hadoop.yarn.api.records.ApplicationReport used on opensource projects

Example 61 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class TestRMRestart method testRMRestartKilledApp.

@Test(timeout = 60000)
public void testRMRestartKilledApp() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();
    Map<ApplicationId, ApplicationStateData> rmAppState = rmState.getApplicationState();
    // start RM
    MockRM rm1 = createMockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // create app and launch the AM
    RMApp app0 = rm1.submitApp(200);
    MockAM am0 = launchAM(app0, rm1, nm1);
    // kill the app.
    rm1.killApp(app0.getApplicationId());
    rm1.waitForState(app0.getApplicationId(), RMAppState.KILLED);
    rm1.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.KILLED);
    // killed state is saved.
    ApplicationStateData appState = rmAppState.get(app0.getApplicationId());
    Assert.assertEquals(RMAppState.KILLED, appState.getState());
    Assert.assertEquals(RMAppAttemptState.KILLED, appState.getAttempt(am0.getApplicationAttemptId()).getState());
    String trackingUrl = app0.getCurrentAppAttempt().getOriginalTrackingUrl();
    Assert.assertNotNull(trackingUrl);
    // restart rm
    MockRM rm2 = createMockRM(conf, memStore);
    rm2.start();
    RMApp loadedApp0 = rm2.getRMContext().getRMApps().get(app0.getApplicationId());
    rm2.waitForState(app0.getApplicationId(), RMAppState.KILLED);
    rm2.waitForState(am0.getApplicationAttemptId(), RMAppAttemptState.KILLED);
    // no new attempt is created.
    Assert.assertEquals(1, loadedApp0.getAppAttempts().size());
    ApplicationReport appReport = verifyAppReportAfterRMRestart(app0, rm2);
    Assert.assertEquals(app0.getDiagnostics().toString(), appReport.getDiagnostics());
    Assert.assertEquals(trackingUrl, loadedApp0.getCurrentAppAttempt().getOriginalTrackingUrl());
}

Example 62 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class TestRM method testInvalidatedAMHostPortOnAMRestart.

@Test(timeout = 60000)
public void testInvalidatedAMHostPortOnAMRestart() throws Exception {
    MockRM rm1 = new MockRM(conf);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // a failed app
    RMApp app2 = rm1.submitApp(200);
    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
    nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.FAILED);
    rm1.waitForState(app2.getApplicationId(), RMAppState.ACCEPTED);
    // before new attempt is launched, the app report returns the invalid AM
    // host and port.
    GetApplicationReportRequest request1 = GetApplicationReportRequest.newInstance(app2.getApplicationId());
    ApplicationReport report1 = rm1.getClientRMService().getApplicationReport(request1).getApplicationReport();
    Assert.assertEquals("N/A", report1.getHost());
    Assert.assertEquals(-1, report1.getRpcPort());
}

Example 63 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class TestRM method testInvalidateAMHostPortWhenAMFailedOrKilled.

// This is to test AM Host and rpc port are invalidated after the am attempt
// is killed or failed, so that client doesn't get the wrong information.
@Test(timeout = 80000)
public void testInvalidateAMHostPortWhenAMFailedOrKilled() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
    MockRM rm1 = new MockRM(conf);
    rm1.start();
    // a succeeded app
    RMApp app1 = rm1.submitApp(200);
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    MockAM am1 = MockRM.launchAndRegisterAM(app1, rm1, nm1);
    MockRM.finishAMAndVerifyAppState(app1, rm1, nm1, am1);
    // a failed app
    RMApp app2 = rm1.submitApp(200);
    MockAM am2 = MockRM.launchAndRegisterAM(app2, rm1, nm1);
    nm1.nodeHeartbeat(am2.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.FAILED);
    rm1.waitForState(app2.getApplicationId(), RMAppState.FAILED);
    // a killed app
    RMApp app3 = rm1.submitApp(200);
    MockAM am3 = MockRM.launchAndRegisterAM(app3, rm1, nm1);
    rm1.killApp(app3.getApplicationId());
    rm1.waitForState(app3.getApplicationId(), RMAppState.KILLED);
    rm1.waitForState(am3.getApplicationAttemptId(), RMAppAttemptState.KILLED);
    GetApplicationsRequest request1 = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.FINISHED, YarnApplicationState.KILLED, YarnApplicationState.FAILED));
    GetApplicationsResponse response1 = rm1.getClientRMService().getApplications(request1);
    List<ApplicationReport> appList1 = response1.getApplicationList();
    Assert.assertEquals(3, appList1.size());
    for (ApplicationReport report : appList1) {
        // killed/failed apps host and rpc port are invalidated.
        if (report.getApplicationId().equals(app2.getApplicationId()) || report.getApplicationId().equals(app3.getApplicationId())) {
            Assert.assertEquals("N/A", report.getHost());
            Assert.assertEquals(-1, report.getRpcPort());
        }
        // succeeded app's host and rpc port is not invalidated
        if (report.getApplicationId().equals(app1.getApplicationId())) {
            Assert.assertFalse(report.getHost().equals("N/A"));
            Assert.assertTrue(report.getRpcPort() != -1);
        }
    }
}

Example 64 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class TestRMRestart method testRMRestartSucceededApp.

@Test(timeout = 60000)
public void testRMRestartSucceededApp() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    RMState rmState = memStore.getState();
    Map<ApplicationId, ApplicationStateData> rmAppState = rmState.getApplicationState();
    // start RM
    MockRM rm1 = createMockRM(conf, memStore);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // create an app and finish the app.
    RMApp app0 = rm1.submitApp(200);
    MockAM am0 = launchAM(app0, rm1, nm1);
    // unregister am
    FinishApplicationMasterRequest req = FinishApplicationMasterRequest.newInstance(FinalApplicationStatus.SUCCEEDED, "diagnostics", "trackingUrl");
    finishApplicationMaster(app0, rm1, nm1, am0, req);
    // check the state store about the unregistered info.
    ApplicationStateData appState = rmAppState.get(app0.getApplicationId());
    ApplicationAttemptStateData attemptState0 = appState.getAttempt(am0.getApplicationAttemptId());
    Assert.assertEquals("diagnostics", attemptState0.getDiagnostics());
    Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, attemptState0.getFinalApplicationStatus());
    Assert.assertEquals("trackingUrl", attemptState0.getFinalTrackingUrl());
    Assert.assertEquals(app0.getFinishTime(), appState.getFinishTime());
    // restart rm
    MockRM rm2 = createMockRM(conf, memStore);
    rm2.start();
    // verify application report returns the same app info as the app info
    // before RM restarts.
    ApplicationReport appReport = verifyAppReportAfterRMRestart(app0, rm2);
    Assert.assertEquals(FinalApplicationStatus.SUCCEEDED, appReport.getFinalApplicationStatus());
    Assert.assertEquals("trackingUrl", appReport.getOriginalTrackingUrl());
}

Example 65 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class MockAsm method newApplication.

public static RMApp newApplication(int i) {
    final ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(newAppID(i), 0);
    final Container masterContainer = Records.newRecord(Container.class);
    ContainerId containerId = ContainerId.newContainerId(appAttemptId, 0);
    masterContainer.setId(containerId);
    masterContainer.setNodeHttpAddress("node:port");
    final String user = newUserName();
    final String name = newAppName();
    final String queue = newQueue();
    final long start = 123456 + i * 1000;
    final long finish = 234567 + i * 1000;
    final String type = YarnConfiguration.DEFAULT_APPLICATION_TYPE;
    YarnApplicationState[] allStates = YarnApplicationState.values();
    final YarnApplicationState state = allStates[i % allStates.length];
    final int maxAppAttempts = i % 1000;
    return new ApplicationBase() {

        @Override
        public ApplicationId getApplicationId() {
            return appAttemptId.getApplicationId();
        }

        @Override
        public String getUser() {
            return user;
        }

        @Override
        public String getName() {
            return name;
        }

        @Override
        public String getApplicationType() {
            return type;
        }

        @Override
        public String getQueue() {
            return queue;
        }

        @Override
        public long getStartTime() {
            return start;
        }

        @Override
        public long getFinishTime() {
            return finish;
        }

        @Override
        public String getTrackingUrl() {
            return null;
        }

        @Override
        public YarnApplicationState createApplicationState() {
            return state;
        }

        @Override
        public StringBuilder getDiagnostics() {
            return new StringBuilder();
        }

        @Override
        public float getProgress() {
            return (float) Math.random();
        }

        @Override
        public FinalApplicationStatus getFinalApplicationStatus() {
            return FinalApplicationStatus.UNDEFINED;
        }

        @Override
        public RMAppAttempt getCurrentAppAttempt() {
            return null;
        }

        @Override
        public int getMaxAppAttempts() {
            return maxAppAttempts;
        }

        @Override
        public Set<String> getApplicationTags() {
            return null;
        }

        @Override
        public ApplicationReport createAndGetApplicationReport(String clientUserName, boolean allowAccess) {
            ApplicationResourceUsageReport usageReport = ApplicationResourceUsageReport.newInstance(0, 0, null, null, null, 0, 0, 0, 0, 0, 0);
            ApplicationReport report = ApplicationReport.newInstance(getApplicationId(), appAttemptId, getUser(), getQueue(), getName(), null, 0, null, null, getDiagnostics().toString(), getTrackingUrl(), getStartTime(), getFinishTime(), getFinalApplicationStatus(), usageReport, null, getProgress(), type, null);
            return report;
        }
    };
}