Examples with LogHandlerAppStartedEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent used on opensource projects

Example 11 with LogHandlerAppStartedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent in project hadoop by apache.

the class LogAggregationService method handle.

@Override
public void handle(LogHandlerEvent event) {
    switch(event.getType()) {
        case APPLICATION_STARTED:
            LogHandlerAppStartedEvent appStartEvent = (LogHandlerAppStartedEvent) event;
            initApp(appStartEvent.getApplicationId(), appStartEvent.getUser(), appStartEvent.getCredentials(), appStartEvent.getApplicationAcls(), appStartEvent.getLogAggregationContext(), appStartEvent.getRecoveredAppLogInitedTime());
            break;
        case CONTAINER_FINISHED:
            LogHandlerContainerFinishedEvent containerFinishEvent = (LogHandlerContainerFinishedEvent) event;
            stopContainer(containerFinishEvent.getContainerId(), containerFinishEvent.getExitCode());
            break;
        case APPLICATION_FINISHED:
            LogHandlerAppFinishedEvent appFinishedEvent = (LogHandlerAppFinishedEvent) event;
            stopApp(appFinishedEvent.getApplicationId());
            break;
        default:
            // Ignore
            ;
    }
}

Example 12 with LogHandlerAppStartedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent in project hadoop by apache.

the class NonAggregatingLogHandler method handle.

@SuppressWarnings("unchecked")
@Override
public void handle(LogHandlerEvent event) {
    switch(event.getType()) {
        case APPLICATION_STARTED:
            LogHandlerAppStartedEvent appStartedEvent = (LogHandlerAppStartedEvent) event;
            this.appOwners.put(appStartedEvent.getApplicationId(), appStartedEvent.getUser());
            this.dispatcher.getEventHandler().handle(new ApplicationEvent(appStartedEvent.getApplicationId(), ApplicationEventType.APPLICATION_LOG_HANDLING_INITED));
            break;
        case CONTAINER_FINISHED:
            // Ignore
            break;
        case APPLICATION_FINISHED:
            LogHandlerAppFinishedEvent appFinishedEvent = (LogHandlerAppFinishedEvent) event;
            ApplicationId appId = appFinishedEvent.getApplicationId();
            // Schedule - so that logs are available on the UI till they're deleted.
            LOG.info("Scheduling Log Deletion for application: " + appId + ", with delay of " + this.deleteDelaySeconds + " seconds");
            String user = appOwners.remove(appId);
            if (user == null) {
                LOG.error("Unable to locate user for " + appId);
                // send LOG_HANDLING_FAILED out
                NonAggregatingLogHandler.this.dispatcher.getEventHandler().handle(new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED));
                break;
            }
            LogDeleterRunnable logDeleter = new LogDeleterRunnable(user, appId);
            long deletionTimestamp = System.currentTimeMillis() + this.deleteDelaySeconds * 1000;
            LogDeleterProto deleterProto = LogDeleterProto.newBuilder().setUser(user).setDeletionTime(deletionTimestamp).build();
            try {
                stateStore.storeLogDeleter(appId, deleterProto);
            } catch (IOException e) {
                LOG.error("Unable to record log deleter state", e);
            }
            try {
                sched.schedule(logDeleter, this.deleteDelaySeconds, TimeUnit.SECONDS);
            } catch (RejectedExecutionException e) {
                // Handling this event in local thread before starting threads
                // or after calling sched.shutdownNow().
                logDeleter.run();
            }
            break;
        default:
            // Ignore
            ;
    }
}

Example 13 with LogHandlerAppStartedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationCreateDirsFailsWithoutKillingNM.

@Test
public void testLogAggregationCreateDirsFailsWithoutKillingNM() throws Exception {
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    DeletionService spyDelSrvc = spy(this.delSrvc);
    LogAggregationService logAggregationService = spy(new LogAggregationService(dispatcher, this.context, spyDelSrvc, super.dirsHandler));
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId appId = BuilderUtils.newApplicationId(System.currentTimeMillis(), (int) (Math.random() * 1000));
    File appLogDir = new File(localLogDir, appId.toString());
    appLogDir.mkdir();
    Exception e = new RuntimeException("KABOOM!");
    doThrow(e).when(logAggregationService).createAppDir(any(String.class), any(ApplicationId.class), any(UserGroupInformation.class));
    LogAggregationContext contextWithAMAndFailed = Records.newRecord(LogAggregationContext.class);
    contextWithAMAndFailed.setLogAggregationPolicyClassName(AMOrFailedContainerLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, contextWithAMAndFailed));
    dispatcher.await();
    ApplicationEvent[] expectedEvents = new ApplicationEvent[] { new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED) };
    checkEvents(appEventHandler, expectedEvents, false, "getType", "getApplicationID", "getDiagnostic");
    // verify trying to collect logs for containers/apps we don't know about
    // doesn't blow up and tear down the NM
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(BuilderUtils.newContainerId(4, 1, 1, 1), 0));
    dispatcher.await();
    logAggregationService.handle(new LogHandlerAppFinishedEvent(BuilderUtils.newApplicationId(1, 5)));
    dispatcher.await();
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
    // local log dir shouldn't be deleted given log aggregation cannot
    // continue due to aggregated log dir creation failure on remoteFS.
    verify(spyDelSrvc, never()).delete(eq(user), any(Path.class), Mockito.<Path>anyVararg());
    verify(logAggregationService).closeFileSystems(any(UserGroupInformation.class));
    // make sure local log dir is not deleted in case log aggregation
    // service cannot be initiated.
    assertTrue(appLogDir.exists());
}

Example 14 with LogHandlerAppStartedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationInitAppFailsWithoutKillingNM.

@Test
@SuppressWarnings("unchecked")
public void testLogAggregationInitAppFailsWithoutKillingNM() throws Exception {
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    LogAggregationService logAggregationService = spy(new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler));
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId appId = BuilderUtils.newApplicationId(System.currentTimeMillis(), (int) (Math.random() * 1000));
    doThrow(new YarnRuntimeException("KABOOM!")).when(logAggregationService).initAppAggregator(eq(appId), eq(user), any(Credentials.class), anyMap(), any(LogAggregationContext.class), anyLong());
    LogAggregationContext contextWithAMAndFailed = Records.newRecord(LogAggregationContext.class);
    contextWithAMAndFailed.setLogAggregationPolicyClassName(AMOrFailedContainerLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, contextWithAMAndFailed));
    dispatcher.await();
    ApplicationEvent[] expectedEvents = new ApplicationEvent[] { new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED) };
    checkEvents(appEventHandler, expectedEvents, false, "getType", "getApplicationID", "getDiagnostic");
    // no filesystems instantiated yet
    verify(logAggregationService, never()).closeFileSystems(any(UserGroupInformation.class));
    // verify trying to collect logs for containers/apps we don't know about
    // doesn't blow up and tear down the NM
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(BuilderUtils.newContainerId(4, 1, 1, 1), 0));
    dispatcher.await();
    logAggregationService.handle(new LogHandlerAppFinishedEvent(BuilderUtils.newApplicationId(1, 5)));
    dispatcher.await();
}

Example 15 with LogHandlerAppStartedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent in project hadoop by apache.

the class TestLogAggregationService method testAppLogDirCreation.

@Test
public void testAppLogDirCreation() throws Exception {
    final String logSuffix = "logs";
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR_SUFFIX, logSuffix);
    InlineDispatcher dispatcher = new InlineDispatcher();
    dispatcher.init(this.conf);
    dispatcher.start();
    FileSystem fs = FileSystem.get(this.conf);
    final FileSystem spyFs = spy(FileSystem.get(this.conf));
    LogAggregationService aggSvc = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler) {

        @Override
        protected FileSystem getFileSystem(Configuration conf) {
            return spyFs;
        }
    };
    aggSvc.init(this.conf);
    aggSvc.start();
    // start an application and verify user, suffix, and app dirs created
    ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
    Path userDir = fs.makeQualified(new Path(remoteRootLogDir.getAbsolutePath(), this.user));
    Path suffixDir = new Path(userDir, logSuffix);
    Path appDir = new Path(suffixDir, appId.toString());
    LogAggregationContext contextWithAllContainers = Records.newRecord(LogAggregationContext.class);
    contextWithAllContainers.setLogAggregationPolicyClassName(AllContainerLogAggregationPolicy.class.getName());
    aggSvc.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, contextWithAllContainers));
    verify(spyFs).mkdirs(eq(userDir), isA(FsPermission.class));
    verify(spyFs).mkdirs(eq(suffixDir), isA(FsPermission.class));
    verify(spyFs).mkdirs(eq(appDir), isA(FsPermission.class));
    // start another application and verify only app dir created
    ApplicationId appId2 = BuilderUtils.newApplicationId(1, 2);
    Path appDir2 = new Path(suffixDir, appId2.toString());
    aggSvc.handle(new LogHandlerAppStartedEvent(appId2, this.user, null, this.acls, contextWithAllContainers));
    verify(spyFs).mkdirs(eq(appDir2), isA(FsPermission.class));
    // start another application with the app dir already created and verify
    // we do not try to create it again
    ApplicationId appId3 = BuilderUtils.newApplicationId(1, 3);
    Path appDir3 = new Path(suffixDir, appId3.toString());
    new File(appDir3.toUri().getPath()).mkdir();
    aggSvc.handle(new LogHandlerAppStartedEvent(appId3, this.user, null, this.acls, contextWithAllContainers));
    verify(spyFs, never()).mkdirs(eq(appDir3), isA(FsPermission.class));
    aggSvc.stop();
    aggSvc.close();
    dispatcher.stop();
}