Examples with LogHandlerAppStartedEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent used on opensource projects

Example 16 with LogHandlerAppStartedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent in project hadoop by apache.

the class TestLogAggregationService method testNoLogsUploadedOnAppFinish.

/* Test to verify fix for YARN-3793 */
@Test
public void testNoLogsUploadedOnAppFinish() throws Exception {
    this.delSrvc = new DeletionService(createContainerExecutor());
    delSrvc = spy(delSrvc);
    this.delSrvc.init(conf);
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId app = BuilderUtils.newApplicationId(1234, 1);
    File appLogDir = new File(localLogDir, app.toString());
    appLogDir.mkdir();
    LogAggregationContext context = LogAggregationContext.newInstance("HOST*", "sys*");
    logAggregationService.handle(new LogHandlerAppStartedEvent(app, this.user, null, this.acls, context));
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(app, 1);
    ContainerId cont = createContainer(appAttemptId, 1, ContainerType.APPLICATION_MASTER);
    writeContainerLogs(appLogDir, cont, new String[] { "stdout", "stderr", "syslog" });
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(cont, 0));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(app));
    logAggregationService.stop();
    delSrvc.stop();
    // Aggregated logs should not be deleted if not uploaded.
    verify(delSrvc, times(0)).delete(user, null);
}

Example 17 with LogHandlerAppStartedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent in project hadoop by apache.

the class TestNonAggregatingLogHandler method testRecovery.

@Test
public void testRecovery() throws Exception {
    File[] localLogDirs = getLocalLogDirFiles(this.getClass().getName(), 2);
    String localLogDirsString = localLogDirs[0].getAbsolutePath() + "," + localLogDirs[1].getAbsolutePath();
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);
    conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, YarnConfiguration.DEFAULT_NM_LOG_RETAIN_SECONDS);
    dirsHandler.init(conf);
    appEventHandler.resetLogHandlingEvent();
    assertFalse(appEventHandler.receiveLogHandlingFinishEvent());
    NMStateStoreService stateStore = new NMMemoryStateStoreService();
    stateStore.init(conf);
    stateStore.start();
    NonAggregatingLogHandlerWithMockExecutor logHandler = new NonAggregatingLogHandlerWithMockExecutor(dispatcher, mockDelService, dirsHandler, stateStore);
    logHandler.init(conf);
    logHandler.start();
    logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null, null));
    logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));
    logHandler.handle(new LogHandlerAppFinishedEvent(appId));
    // simulate a restart and verify deletion is rescheduled
    logHandler.close();
    logHandler = new NonAggregatingLogHandlerWithMockExecutor(dispatcher, mockDelService, dirsHandler, stateStore);
    logHandler.init(conf);
    logHandler.start();
    ArgumentCaptor<Runnable> schedArg = ArgumentCaptor.forClass(Runnable.class);
    verify(logHandler.mockSched).schedule(schedArg.capture(), anyLong(), eq(TimeUnit.MILLISECONDS));
    // execute the runnable and verify another restart has nothing scheduled
    schedArg.getValue().run();
    logHandler.close();
    logHandler = new NonAggregatingLogHandlerWithMockExecutor(dispatcher, mockDelService, dirsHandler, stateStore);
    logHandler.init(conf);
    logHandler.start();
    verify(logHandler.mockSched, never()).schedule(any(Runnable.class), anyLong(), any(TimeUnit.class));
    // wait events get drained.
    this.dispatcher.await();
    assertTrue(appEventHandler.receiveLogHandlingFinishEvent());
    appEventHandler.resetLogHandlingEvent();
    assertFalse(appEventHandler.receiveLogHandlingFailedEvent());
    // send an app finish event against a removed app
    logHandler.handle(new LogHandlerAppFinishedEvent(appId));
    this.dispatcher.await();
    // verify to receive a log failed event.
    assertTrue(appEventHandler.receiveLogHandlingFailedEvent());
    assertFalse(appEventHandler.receiveLogHandlingFinishEvent());
    logHandler.close();
}

Example 18 with LogHandlerAppStartedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationServiceWithPatternsAndIntervals.

@SuppressWarnings("resource")
@Test(timeout = 50000)
public void testLogAggregationServiceWithPatternsAndIntervals() throws Exception {
    LogAggregationContext logAggregationContext = Records.newRecord(LogAggregationContext.class);
    // set IncludePattern and RolledLogsIncludePattern.
    // When the app is running, we only aggregate the log with
    // the name stdout. After the app finishes, we only aggregate
    // the log with the name std_final.
    logAggregationContext.setRolledLogsIncludePattern("stdout");
    logAggregationContext.setIncludePattern("std_final");
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    //configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to
    //have fully qualified path
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.toURI().toString());
    this.conf.setLong(YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, 3600);
    this.conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600);
    ApplicationId application = BuilderUtils.newApplicationId(System.currentTimeMillis(), 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(application, 1);
    ContainerId container = createContainer(appAttemptId, 1, ContainerType.APPLICATION_MASTER);
    ConcurrentMap<ApplicationId, Application> maps = this.context.getApplications();
    Application app = mock(Application.class);
    maps.put(application, app);
    when(app.getContainers()).thenReturn(this.context.getContainers());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    // AppLogDir should be created
    File appLogDir = new File(localLogDir, ConverterUtils.toString(application));
    appLogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application, this.user, null, this.acls, logAggregationContext));
    // Simulate log-file creation
    // create std_final in log directory which will not be aggregated
    // until the app finishes.
    String[] logFilesWithFinalLog = new String[] { "stdout", "std_final" };
    writeContainerLogs(appLogDir, container, logFilesWithFinalLog);
    // Do log aggregation
    AppLogAggregatorImpl aggregator = (AppLogAggregatorImpl) logAggregationService.getAppLogAggregators().get(application);
    aggregator.doLogAggregationOutOfBand();
    Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, false, null));
    String[] logFiles = new String[] { "stdout" };
    verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles, 1, true);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container, 0));
    dispatcher.await();
    // Do the log aggregation after ContainerFinishedEvent but before
    // AppFinishedEvent. The std_final is expected to be aggregated this time
    // even if the app is running but the container finishes.
    aggregator.doLogAggregationOutOfBand();
    Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 2, false, null));
    // This container finishes.
    // The log "std_final" should be aggregated this time.
    String[] logFinalLog = new String[] { "std_final" };
    verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFinalLog, 1, true);
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application));
    logAggregationService.stop();
}

Example 19 with LogHandlerAppStartedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent in project hadoop by apache.

the class TestLogAggregationService method testMultipleAppsLogAggregation.

@Test
@SuppressWarnings("unchecked")
public void testMultipleAppsLogAggregation() throws Exception {
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    String[] fileNames = new String[] { "stdout", "stderr", "syslog" };
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    // AppLogDir should be created
    File app1LogDir = new File(localLogDir, application1.toString());
    app1LogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application1, this.user, null, this.acls));
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(application1, 1);
    ContainerId container11 = createContainer(appAttemptId1, 1, ContainerType.APPLICATION_MASTER);
    // Simulate log-file creation
    writeContainerLogs(app1LogDir, container11, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container11, 0));
    ApplicationId application2 = BuilderUtils.newApplicationId(1234, 2);
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(application2, 1);
    File app2LogDir = new File(localLogDir, application2.toString());
    app2LogDir.mkdir();
    LogAggregationContext contextWithAMOnly = Records.newRecord(LogAggregationContext.class);
    contextWithAMOnly.setLogAggregationPolicyClassName(AMOnlyLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(application2, this.user, null, this.acls, contextWithAMOnly));
    ContainerId container21 = createContainer(appAttemptId2, 1, ContainerType.APPLICATION_MASTER);
    writeContainerLogs(app2LogDir, container21, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container21, 0));
    ContainerId container12 = createContainer(appAttemptId1, 2, ContainerType.TASK);
    writeContainerLogs(app1LogDir, container12, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container12, 0));
    ApplicationId application3 = BuilderUtils.newApplicationId(1234, 3);
    ApplicationAttemptId appAttemptId3 = BuilderUtils.newApplicationAttemptId(application3, 1);
    File app3LogDir = new File(localLogDir, application3.toString());
    app3LogDir.mkdir();
    LogAggregationContext contextWithAMAndFailed = Records.newRecord(LogAggregationContext.class);
    contextWithAMAndFailed.setLogAggregationPolicyClassName(AMOrFailedContainerLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(application3, this.user, null, this.acls, contextWithAMAndFailed));
    dispatcher.await();
    ApplicationEvent[] expectedInitEvents = new ApplicationEvent[] { new ApplicationEvent(application1, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED), new ApplicationEvent(application2, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED), new ApplicationEvent(application3, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED) };
    checkEvents(appEventHandler, expectedInitEvents, false, "getType", "getApplicationID");
    reset(appEventHandler);
    ContainerId container31 = createContainer(appAttemptId3, 1, ContainerType.APPLICATION_MASTER);
    writeContainerLogs(app3LogDir, container31, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container31, 0));
    ContainerId container32 = createContainer(appAttemptId3, 2, ContainerType.TASK);
    writeContainerLogs(app3LogDir, container32, fileNames);
    logAggregationService.handle(// Failed 
    new LogHandlerContainerFinishedEvent(container32, 1));
    ContainerId container22 = createContainer(appAttemptId2, 2, ContainerType.TASK);
    writeContainerLogs(app2LogDir, container22, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container22, 0));
    ContainerId container33 = createContainer(appAttemptId3, 3, ContainerType.TASK);
    writeContainerLogs(app3LogDir, container33, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container33, 0));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application2));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application3));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application1));
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
    verifyContainerLogs(logAggregationService, application1, new ContainerId[] { container11, container12 }, fileNames, 3, false);
    verifyContainerLogs(logAggregationService, application2, new ContainerId[] { container21 }, fileNames, 3, false);
    verifyContainerLogs(logAggregationService, application3, new ContainerId[] { container31, container32 }, fileNames, 3, false);
    dispatcher.await();
    ApplicationEvent[] expectedFinishedEvents = new ApplicationEvent[] { new ApplicationEvent(application1, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED), new ApplicationEvent(application2, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED), new ApplicationEvent(application3, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED) };
    checkEvents(appEventHandler, expectedFinishedEvents, false, "getType", "getApplicationID");
}

Example 20 with LogHandlerAppStartedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent in project hadoop by apache.

the class TestLogAggregationService method testLogAggregatorCleanup.

@Test
public void testLogAggregatorCleanup() throws Exception {
    DeletionService delSrvc = mock(DeletionService.class);
    // get the AppLogAggregationImpl thread to crash
    LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, delSrvc, mockedDirSvc);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    logAggregationService.handle(new LogHandlerAppStartedEvent(application1, this.user, null, this.acls));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application1));
    dispatcher.await();
    int timeToWait = 20 * 1000;
    while (timeToWait > 0 && logAggregationService.getNumAggregators() > 0) {
        Thread.sleep(100);
        timeToWait -= 100;
    }
    Assert.assertEquals("Log aggregator failed to cleanup!", 0, logAggregationService.getNumAggregators());
    logAggregationService.stop();
    logAggregationService.close();
}