Examples with LogHandlerContainerFinishedEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent used on opensource projects

Example 6 with LogHandlerContainerFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent in project hadoop by apache.

the class LogAggregationService method handle.

@Override
public void handle(LogHandlerEvent event) {
    switch(event.getType()) {
        case APPLICATION_STARTED:
            LogHandlerAppStartedEvent appStartEvent = (LogHandlerAppStartedEvent) event;
            initApp(appStartEvent.getApplicationId(), appStartEvent.getUser(), appStartEvent.getCredentials(), appStartEvent.getApplicationAcls(), appStartEvent.getLogAggregationContext(), appStartEvent.getRecoveredAppLogInitedTime());
            break;
        case CONTAINER_FINISHED:
            LogHandlerContainerFinishedEvent containerFinishEvent = (LogHandlerContainerFinishedEvent) event;
            stopContainer(containerFinishEvent.getContainerId(), containerFinishEvent.getExitCode());
            break;
        case APPLICATION_FINISHED:
            LogHandlerAppFinishedEvent appFinishedEvent = (LogHandlerAppFinishedEvent) event;
            stopApp(appFinishedEvent.getApplicationId());
            break;
        default:
            // Ignore
            ;
    }
}

Example 7 with LogHandlerContainerFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent in project hadoop by apache.

the class ContainerImpl method sendFinishedEvents.

@SuppressWarnings("unchecked")
private void sendFinishedEvents() {
    // Inform the application
    @SuppressWarnings("rawtypes") EventHandler eventHandler = dispatcher.getEventHandler();
    ContainerStatus containerStatus = cloneAndGetContainerStatus();
    eventHandler.handle(new ApplicationContainerFinishedEvent(containerStatus));
    // Tell the scheduler the container is Done
    eventHandler.handle(new ContainerSchedulerEvent(this, ContainerSchedulerEventType.CONTAINER_COMPLETED));
    // Remove the container from the resource-monitor
    eventHandler.handle(new ContainerStopMonitoringEvent(containerId));
    // Tell the logService too
    eventHandler.handle(new LogHandlerContainerFinishedEvent(containerId, exitCode));
}

Example 8 with LogHandlerContainerFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationCreateDirsFailsWithoutKillingNM.

@Test
public void testLogAggregationCreateDirsFailsWithoutKillingNM() throws Exception {
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    DeletionService spyDelSrvc = spy(this.delSrvc);
    LogAggregationService logAggregationService = spy(new LogAggregationService(dispatcher, this.context, spyDelSrvc, super.dirsHandler));
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId appId = BuilderUtils.newApplicationId(System.currentTimeMillis(), (int) (Math.random() * 1000));
    File appLogDir = new File(localLogDir, appId.toString());
    appLogDir.mkdir();
    Exception e = new RuntimeException("KABOOM!");
    doThrow(e).when(logAggregationService).createAppDir(any(String.class), any(ApplicationId.class), any(UserGroupInformation.class));
    LogAggregationContext contextWithAMAndFailed = Records.newRecord(LogAggregationContext.class);
    contextWithAMAndFailed.setLogAggregationPolicyClassName(AMOrFailedContainerLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, contextWithAMAndFailed));
    dispatcher.await();
    ApplicationEvent[] expectedEvents = new ApplicationEvent[] { new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED) };
    checkEvents(appEventHandler, expectedEvents, false, "getType", "getApplicationID", "getDiagnostic");
    // verify trying to collect logs for containers/apps we don't know about
    // doesn't blow up and tear down the NM
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(BuilderUtils.newContainerId(4, 1, 1, 1), 0));
    dispatcher.await();
    logAggregationService.handle(new LogHandlerAppFinishedEvent(BuilderUtils.newApplicationId(1, 5)));
    dispatcher.await();
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
    // local log dir shouldn't be deleted given log aggregation cannot
    // continue due to aggregated log dir creation failure on remoteFS.
    verify(spyDelSrvc, never()).delete(eq(user), any(Path.class), Mockito.<Path>anyVararg());
    verify(logAggregationService).closeFileSystems(any(UserGroupInformation.class));
    // make sure local log dir is not deleted in case log aggregation
    // service cannot be initiated.
    assertTrue(appLogDir.exists());
}

Example 9 with LogHandlerContainerFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationInitAppFailsWithoutKillingNM.

@Test
@SuppressWarnings("unchecked")
public void testLogAggregationInitAppFailsWithoutKillingNM() throws Exception {
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    LogAggregationService logAggregationService = spy(new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler));
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId appId = BuilderUtils.newApplicationId(System.currentTimeMillis(), (int) (Math.random() * 1000));
    doThrow(new YarnRuntimeException("KABOOM!")).when(logAggregationService).initAppAggregator(eq(appId), eq(user), any(Credentials.class), anyMap(), any(LogAggregationContext.class), anyLong());
    LogAggregationContext contextWithAMAndFailed = Records.newRecord(LogAggregationContext.class);
    contextWithAMAndFailed.setLogAggregationPolicyClassName(AMOrFailedContainerLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, contextWithAMAndFailed));
    dispatcher.await();
    ApplicationEvent[] expectedEvents = new ApplicationEvent[] { new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED) };
    checkEvents(appEventHandler, expectedEvents, false, "getType", "getApplicationID", "getDiagnostic");
    // no filesystems instantiated yet
    verify(logAggregationService, never()).closeFileSystems(any(UserGroupInformation.class));
    // verify trying to collect logs for containers/apps we don't know about
    // doesn't blow up and tear down the NM
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(BuilderUtils.newContainerId(4, 1, 1, 1), 0));
    dispatcher.await();
    logAggregationService.handle(new LogHandlerAppFinishedEvent(BuilderUtils.newApplicationId(1, 5)));
    dispatcher.await();
}

Example 10 with LogHandlerContainerFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent in project hadoop by apache.

the class TestLogAggregationService method testNoLogsUploadedOnAppFinish.

/* Test to verify fix for YARN-3793 */
@Test
public void testNoLogsUploadedOnAppFinish() throws Exception {
    this.delSrvc = new DeletionService(createContainerExecutor());
    delSrvc = spy(delSrvc);
    this.delSrvc.init(conf);
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId app = BuilderUtils.newApplicationId(1234, 1);
    File appLogDir = new File(localLogDir, app.toString());
    appLogDir.mkdir();
    LogAggregationContext context = LogAggregationContext.newInstance("HOST*", "sys*");
    logAggregationService.handle(new LogHandlerAppStartedEvent(app, this.user, null, this.acls, context));
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(app, 1);
    ContainerId cont = createContainer(appAttemptId, 1, ContainerType.APPLICATION_MASTER);
    writeContainerLogs(appLogDir, cont, new String[] { "stdout", "stderr", "syslog" });
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(cont, 0));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(app));
    logAggregationService.stop();
    delSrvc.stop();
    // Aggregated logs should not be deleted if not uploaded.
    verify(delSrvc, times(0)).delete(user, null);
}