Examples with ApplicationReport org.apache.hadoop.yarn.api.records.ApplicationReport used on opensource projects

Example 86 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class TestClientToAMTokens method testClientTokenRace.

@Test(timeout = 20000)
public void testClientTokenRace() throws Exception {
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    ContainerManagementProtocol containerManager = mock(ContainerManagementProtocol.class);
    StartContainersResponse mockResponse = mock(StartContainersResponse.class);
    when(containerManager.startContainers((StartContainersRequest) any())).thenReturn(mockResponse);
    final DrainDispatcher dispatcher = new DrainDispatcher();
    MockRM rm = new MockRMWithCustomAMLauncher(conf, containerManager) {

        protected ClientRMService createClientRMService() {
            return new ClientRMService(this.rmContext, scheduler, this.rmAppManager, this.applicationACLsManager, this.queueACLsManager, getRMContext().getRMDelegationTokenSecretManager());
        }

        ;

        @Override
        protected Dispatcher createDispatcher() {
            return dispatcher;
        }

        @Override
        protected void doSecureLogin() throws IOException {
        }
    };
    rm.start();
    // Submit an app
    RMApp app = rm.submitApp(1024);
    // Set up a node.
    MockNM nm1 = rm.registerNode("localhost:1234", 3072);
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    nm1.nodeHeartbeat(true);
    dispatcher.await();
    ApplicationAttemptId appAttempt = app.getCurrentAppAttempt().getAppAttemptId();
    final MockAM mockAM = new MockAM(rm.getRMContext(), rm.getApplicationMasterService(), app.getCurrentAppAttempt().getAppAttemptId());
    UserGroupInformation appUgi = UserGroupInformation.createRemoteUser(appAttempt.toString());
    RegisterApplicationMasterResponse response = appUgi.doAs(new PrivilegedAction<RegisterApplicationMasterResponse>() {

        @Override
        public RegisterApplicationMasterResponse run() {
            RegisterApplicationMasterResponse response = null;
            try {
                response = mockAM.registerAppAttempt();
            } catch (Exception e) {
                Assert.fail("Exception was not expected");
            }
            return response;
        }
    });
    // Get the app-report.
    GetApplicationReportRequest request = Records.newRecord(GetApplicationReportRequest.class);
    request.setApplicationId(app.getApplicationId());
    GetApplicationReportResponse reportResponse = rm.getClientRMService().getApplicationReport(request);
    ApplicationReport appReport = reportResponse.getApplicationReport();
    org.apache.hadoop.yarn.api.records.Token originalClientToAMToken = appReport.getClientToAMToken();
    // ClientToAMToken master key should have been received on register
    // application master response.
    final ByteBuffer clientMasterKey = response.getClientToAMTokenMasterKey();
    Assert.assertNotNull(clientMasterKey);
    Assert.assertTrue(clientMasterKey.array().length > 0);
    // Start the AM with the correct shared-secret.
    ApplicationAttemptId appAttemptId = app.getAppAttempts().keySet().iterator().next();
    Assert.assertNotNull(appAttemptId);
    final CustomAM am = new CustomAM(appAttemptId, null);
    am.init(conf);
    am.start();
    // Now the real test!
    // Set up clients to be able to pick up correct tokens.
    SecurityUtil.setSecurityInfoProviders(new CustomSecurityInfo());
    Token<ClientToAMTokenIdentifier> token = ConverterUtils.convertFromYarn(originalClientToAMToken, am.address);
    // Schedule the key to be set after a significant delay
    Timer timer = new Timer();
    TimerTask timerTask = new TimerTask() {

        @Override
        public void run() {
            am.setClientSecretKey(clientMasterKey.array());
        }
    };
    timer.schedule(timerTask, 250);
    // connect should pause waiting for the master key to arrive
    verifyValidToken(conf, am, token);
    am.stop();
    rm.stop();
}

Example 87 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class TestRMWebApp method mockClientRMService.

public static ClientRMService mockClientRMService(RMContext rmContext) {
    ClientRMService clientRMService = mock(ClientRMService.class);
    List<ApplicationReport> appReports = new ArrayList<ApplicationReport>();
    for (RMApp app : rmContext.getRMApps().values()) {
        ApplicationReport appReport = ApplicationReport.newInstance(app.getApplicationId(), (ApplicationAttemptId) null, app.getUser(), app.getQueue(), app.getName(), (String) null, 0, (Token) null, app.createApplicationState(), app.getDiagnostics().toString(), (String) null, app.getStartTime(), app.getFinishTime(), app.getFinalApplicationStatus(), (ApplicationResourceUsageReport) null, app.getTrackingUrl(), app.getProgress(), app.getApplicationType(), (Token) null);
        appReports.add(appReport);
    }
    GetApplicationsResponse response = mock(GetApplicationsResponse.class);
    when(response.getApplicationList()).thenReturn(appReports);
    try {
        when(clientRMService.getApplications(any(GetApplicationsRequest.class))).thenReturn(response);
    } catch (YarnException e) {
        Assert.fail("Exception is not expected.");
    }
    return clientRMService;
}

Example 88 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class TestSubmitApplicationWithRMHA method testHandleRMHAafterSubmitApplicationCallWithSavedApplicationState.

// There are two scenarios when RM failover happens
// after SubmitApplication Call:
// 1) RMStateStore already saved the ApplicationState when failover happens
// 2) RMStateStore did not save the ApplicationState when failover happens
@Test
public void testHandleRMHAafterSubmitApplicationCallWithSavedApplicationState() throws Exception {
    // Test scenario 1 when RM failover happens
    // after SubmitApplication Call:
    // RMStateStore already saved the ApplicationState when failover happens
    startRMs();
    // Submit Application
    // After submission, the applicationState will be saved in RMStateStore.
    RMApp app0 = rm1.submitApp(200);
    // Do the failover
    explicitFailover();
    // Since the applicationState has already been saved in RMStateStore
    // before failover happens, the current active rm can load the previous
    // applicationState.
    ApplicationReport appReport = rm2.getApplicationReport(app0.getApplicationId());
    // verify previous submission is successful.
    Assert.assertTrue(appReport.getYarnApplicationState() == YarnApplicationState.ACCEPTED || appReport.getYarnApplicationState() == YarnApplicationState.SUBMITTED);
}

Example 89 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class TestRMRestart method testRMRestartGetApplicationList.

@Test(timeout = 60000)
public void testRMRestartGetApplicationList() throws Exception {
    conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS, 1);
    MemoryRMStateStore memStore = new MemoryRMStateStore();
    memStore.init(conf);
    // start RM
    MockRM rm1 = new MockRM(conf, memStore) {

        @Override
        protected SystemMetricsPublisher createSystemMetricsPublisher() {
            return spy(super.createSystemMetricsPublisher());
        }
    };
    rms.add(rm1);
    rm1.start();
    MockNM nm1 = new MockNM("127.0.0.1:1234", 15120, rm1.getResourceTrackerService());
    nm1.registerNode();
    // a succeeded app.
    RMApp app0 = rm1.submitApp(200, "name", "user", null, false, "default", 1, null, "myType");
    MockAM am0 = launchAM(app0, rm1, nm1);
    finishApplicationMaster(app0, rm1, nm1, am0);
    // a failed app.
    RMApp app1 = rm1.submitApp(200, "name", "user", null, false, "default", 1, null, "myType");
    MockAM am1 = launchAM(app1, rm1, nm1);
    // fail the AM by sending CONTAINER_FINISHED event without registering.
    nm1.nodeHeartbeat(am1.getApplicationAttemptId(), 1, ContainerState.COMPLETE);
    rm1.waitForState(am1.getApplicationAttemptId(), RMAppAttemptState.FAILED);
    rm1.waitForState(app1.getApplicationId(), RMAppState.FAILED);
    // a killed app.
    RMApp app2 = rm1.submitApp(200, "name", "user", null, false, "default", 1, null, "myType");
    MockAM am2 = launchAM(app2, rm1, nm1);
    rm1.killApp(app2.getApplicationId());
    rm1.waitForState(app2.getApplicationId(), RMAppState.KILLED);
    rm1.waitForState(am2.getApplicationAttemptId(), RMAppAttemptState.KILLED);
    verify(rm1.getRMContext().getSystemMetricsPublisher(), Mockito.times(3)).appCreated(any(RMApp.class), anyLong());
    // restart rm
    MockRM rm2 = new MockRM(conf, memStore) {

        @Override
        protected RMAppManager createRMAppManager() {
            return spy(super.createRMAppManager());
        }

        @Override
        protected SystemMetricsPublisher createSystemMetricsPublisher() {
            return spy(super.createSystemMetricsPublisher());
        }
    };
    rms.add(rm2);
    rm2.start();
    verify(rm2.getRMContext().getSystemMetricsPublisher(), Mockito.times(3)).appCreated(any(RMApp.class), anyLong());
    GetApplicationsRequest request1 = GetApplicationsRequest.newInstance(EnumSet.of(YarnApplicationState.FINISHED, YarnApplicationState.KILLED, YarnApplicationState.FAILED));
    GetApplicationsResponse response1 = rm2.getClientRMService().getApplications(request1);
    List<ApplicationReport> appList1 = response1.getApplicationList();
    // assert all applications exist according to application state after RM
    // restarts.
    boolean forApp0 = false, forApp1 = false, forApp2 = false;
    for (ApplicationReport report : appList1) {
        if (report.getApplicationId().equals(app0.getApplicationId())) {
            Assert.assertEquals(YarnApplicationState.FINISHED, report.getYarnApplicationState());
            forApp0 = true;
        }
        if (report.getApplicationId().equals(app1.getApplicationId())) {
            Assert.assertEquals(YarnApplicationState.FAILED, report.getYarnApplicationState());
            forApp1 = true;
        }
        if (report.getApplicationId().equals(app2.getApplicationId())) {
            Assert.assertEquals(YarnApplicationState.KILLED, report.getYarnApplicationState());
            forApp2 = true;
        }
    }
    Assert.assertTrue(forApp0 && forApp1 && forApp2);
    // assert all applications exist according to application type after RM
    // restarts.
    Set<String> appTypes = new HashSet<String>();
    appTypes.add("myType");
    GetApplicationsRequest request2 = GetApplicationsRequest.newInstance(appTypes);
    GetApplicationsResponse response2 = rm2.getClientRMService().getApplications(request2);
    List<ApplicationReport> appList2 = response2.getApplicationList();
    Assert.assertTrue(3 == appList2.size());
    // check application summary is logged for the completed apps with timeout
    // to make sure APP_COMPLETED events are processed, after RM restart.
    verify(rm2.getRMAppManager(), timeout(1000).times(3)).logApplicationSummary(isA(ApplicationId.class));
}

Example 90 with ApplicationReport

use of org.apache.hadoop.yarn.api.records.ApplicationReport in project hadoop by apache.

the class TestRemoteAppChecker method testRunningApp.

@Test
public void testRunningApp() throws Exception {
    YarnClient client = createCheckerWithMockedClient();
    ApplicationId id = ApplicationId.newInstance(1, 1);
    // create a report and set the state to an active one
    ApplicationReport report = new ApplicationReportPBImpl();
    report.setYarnApplicationState(YarnApplicationState.ACCEPTED);
    doReturn(report).when(client).getApplicationReport(id);
    assertTrue(checker.isApplicationActive(id));
}