Examples with ApplicationId org.apache.hadoop.yarn.api.records.ApplicationId used on opensource projects

Example 86 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class TestLogAggregationService method createApplication.

private ApplicationId createApplication() {
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    ApplicationId appId = BuilderUtils.newApplicationId(1234, 1);
    Application mockApp = mock(Application.class);
    when(mockApp.getContainers()).thenReturn(new HashMap<ContainerId, Container>());
    this.context.getApplications().put(appId, mockApp);
    return appId;
}

Example 87 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationAbsentContainer.

@Test(timeout = 50000)
public void testLogAggregationAbsentContainer() throws Exception {
    ApplicationId appId = createApplication();
    LogAggregationService logAggregationService = createLogAggregationService(appId, FailedOrKilledContainerLogAggregationPolicy.class, null);
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId containerId = BuilderUtils.newContainerId(appAttemptId1, 2l);
    try {
        logAggregationService.handle(new LogHandlerContainerFinishedEvent(containerId, 100));
        assertTrue("Should skip when null containerID", true);
    } catch (Exception e) {
        Assert.assertFalse("Exception not expected should skip null containerid", true);
    }
}

Example 88 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class TestLogAggregationService method testVerifyAndCreateRemoteDirsFailure.

@Test
public void testVerifyAndCreateRemoteDirsFailure() throws Exception {
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    LogAggregationService logAggregationService = spy(new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler));
    logAggregationService.init(this.conf);
    YarnRuntimeException e = new YarnRuntimeException("KABOOM!");
    doThrow(e).when(logAggregationService).verifyAndCreateRemoteLogDir(any(Configuration.class));
    logAggregationService.start();
    // Now try to start an application
    ApplicationId appId = BuilderUtils.newApplicationId(System.currentTimeMillis(), (int) (Math.random() * 1000));
    LogAggregationContext contextWithAMAndFailed = Records.newRecord(LogAggregationContext.class);
    contextWithAMAndFailed.setLogAggregationPolicyClassName(AMOrFailedContainerLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, contextWithAMAndFailed));
    dispatcher.await();
    // Verify that it failed
    ApplicationEvent[] expectedEvents = new ApplicationEvent[] { new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED) };
    checkEvents(appEventHandler, expectedEvents, false, "getType", "getApplicationID", "getDiagnostic");
    Mockito.reset(logAggregationService);
    // Now try to start another one
    ApplicationId appId2 = BuilderUtils.newApplicationId(System.currentTimeMillis(), (int) (Math.random() * 1000));
    File appLogDir = new File(localLogDir, appId2.toString());
    appLogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(appId2, this.user, null, this.acls, contextWithAMAndFailed));
    dispatcher.await();
    // Verify that it worked
    expectedEvents = new ApplicationEvent[] { new // original failure
    ApplicationEvent(// original failure
    appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED), new // success
    ApplicationEvent(// success
    appId2, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED) };
    checkEvents(appEventHandler, expectedEvents, false, "getType", "getApplicationID", "getDiagnostic");
    logAggregationService.stop();
}

Example 89 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class TestLogAggregationService method verifyLocalFileDeletion.

private void verifyLocalFileDeletion(LogAggregationService logAggregationService) throws Exception {
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    // AppLogDir should be created
    File app1LogDir = new File(localLogDir, application1.toString());
    app1LogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application1, this.user, null, this.acls));
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(application1, 1);
    ContainerId container11 = createContainer(appAttemptId, 1, ContainerType.APPLICATION_MASTER);
    // Simulate log-file creation
    writeContainerLogs(app1LogDir, container11, new String[] { "stdout", "stderr", "syslog" });
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container11, 0));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application1));
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
    // ensure filesystems were closed
    verify(logAggregationService).closeFileSystems(any(UserGroupInformation.class));
    verify(delSrvc).delete(eq(user), eq((Path) null), eq(new Path(app1LogDir.getAbsolutePath())));
    String containerIdStr = container11.toString();
    File containerLogDir = new File(app1LogDir, containerIdStr);
    int count = 0;
    int maxAttempts = 50;
    for (String fileType : new String[] { "stdout", "stderr", "syslog" }) {
        File f = new File(containerLogDir, fileType);
        count = 0;
        while ((f.exists()) && (count < maxAttempts)) {
            count++;
            Thread.sleep(100);
        }
        Assert.assertFalse("File [" + f + "] was not deleted", f.exists());
    }
    count = 0;
    while ((app1LogDir.exists()) && (count < maxAttempts)) {
        count++;
        Thread.sleep(100);
    }
    Assert.assertFalse("Directory [" + app1LogDir + "] was not deleted", app1LogDir.exists());
    Path logFilePath = logAggregationService.getRemoteNodeLogFileForApp(application1, this.user);
    Assert.assertTrue("Log file [" + logFilePath + "] not found", new File(logFilePath.toUri().getPath()).exists());
    dispatcher.await();
    ApplicationEvent[] expectedEvents = new ApplicationEvent[] { new ApplicationEvent(appAttemptId.getApplicationId(), ApplicationEventType.APPLICATION_LOG_HANDLING_INITED), new ApplicationEvent(appAttemptId.getApplicationId(), ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED) };
    checkEvents(appEventHandler, expectedEvents, true, "getType", "getApplicationID");
}

Example 90 with ApplicationId

use of org.apache.hadoop.yarn.api.records.ApplicationId in project hadoop by apache.

the class TestLogAggregationService method verifySkipUnnecessaryNNOperations.

private void verifySkipUnnecessaryNNOperations(LogAggregationContext logAggregationContext, int expectedLogAggregationTimes, int expectedAggregationReportNum, int expectedCleanupOldLogsTimes) throws Exception {
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId appId = createApplication();
    logAggregationService.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, logAggregationContext));
    // Container finishes
    String[] logFiles = new String[] { "sysout" };
    finishContainer(appId, logAggregationService, ContainerType.APPLICATION_MASTER, 1, 0, logFiles);
    AppLogAggregatorImpl aggregator = (AppLogAggregatorImpl) logAggregationService.getAppLogAggregators().get(appId);
    aggregator.doLogAggregationOutOfBand();
    Thread.sleep(2000);
    aggregator.doLogAggregationOutOfBand();
    Thread.sleep(2000);
    // App finishes.
    logAggregationService.handle(new LogHandlerAppFinishedEvent(appId));
    logAggregationService.stop();
    assertEquals(expectedLogAggregationTimes, aggregator.getLogAggregationTimes());
    assertEquals(expectedAggregationReportNum, this.context.getLogAggregationStatusForApps().size());
    assertEquals(expectedCleanupOldLogsTimes, aggregator.getCleanupOldLogTimes());
}