Examples with LogAggregationContext org.apache.hadoop.yarn.api.records.LogAggregationContext used on opensource projects

Example 16 with LogAggregationContext

use of org.apache.hadoop.yarn.api.records.LogAggregationContext in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationServiceWithPatternsAndIntervals.

@SuppressWarnings("resource")
@Test(timeout = 50000)
public void testLogAggregationServiceWithPatternsAndIntervals() throws Exception {
    LogAggregationContext logAggregationContext = Records.newRecord(LogAggregationContext.class);
    // set IncludePattern and RolledLogsIncludePattern.
    // When the app is running, we only aggregate the log with
    // the name stdout. After the app finishes, we only aggregate
    // the log with the name std_final.
    logAggregationContext.setRolledLogsIncludePattern("stdout");
    logAggregationContext.setIncludePattern("std_final");
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    //configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to
    //have fully qualified path
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.toURI().toString());
    this.conf.setLong(YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, 3600);
    this.conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600);
    ApplicationId application = BuilderUtils.newApplicationId(System.currentTimeMillis(), 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(application, 1);
    ContainerId container = createContainer(appAttemptId, 1, ContainerType.APPLICATION_MASTER);
    ConcurrentMap<ApplicationId, Application> maps = this.context.getApplications();
    Application app = mock(Application.class);
    maps.put(application, app);
    when(app.getContainers()).thenReturn(this.context.getContainers());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    // AppLogDir should be created
    File appLogDir = new File(localLogDir, ConverterUtils.toString(application));
    appLogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application, this.user, null, this.acls, logAggregationContext));
    // Simulate log-file creation
    // create std_final in log directory which will not be aggregated
    // until the app finishes.
    String[] logFilesWithFinalLog = new String[] { "stdout", "std_final" };
    writeContainerLogs(appLogDir, container, logFilesWithFinalLog);
    // Do log aggregation
    AppLogAggregatorImpl aggregator = (AppLogAggregatorImpl) logAggregationService.getAppLogAggregators().get(application);
    aggregator.doLogAggregationOutOfBand();
    Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, false, null));
    String[] logFiles = new String[] { "stdout" };
    verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles, 1, true);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container, 0));
    dispatcher.await();
    // Do the log aggregation after ContainerFinishedEvent but before
    // AppFinishedEvent. The std_final is expected to be aggregated this time
    // even if the app is running but the container finishes.
    aggregator.doLogAggregationOutOfBand();
    Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 2, false, null));
    // This container finishes.
    // The log "std_final" should be aggregated this time.
    String[] logFinalLog = new String[] { "std_final" };
    verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFinalLog, 1, true);
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application));
    logAggregationService.stop();
}

Example 17 with LogAggregationContext

use of org.apache.hadoop.yarn.api.records.LogAggregationContext in project hadoop by apache.

the class TestLogAggregationService method testMultipleAppsLogAggregation.

@Test
@SuppressWarnings("unchecked")
public void testMultipleAppsLogAggregation() throws Exception {
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    String[] fileNames = new String[] { "stdout", "stderr", "syslog" };
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    // AppLogDir should be created
    File app1LogDir = new File(localLogDir, application1.toString());
    app1LogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application1, this.user, null, this.acls));
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(application1, 1);
    ContainerId container11 = createContainer(appAttemptId1, 1, ContainerType.APPLICATION_MASTER);
    // Simulate log-file creation
    writeContainerLogs(app1LogDir, container11, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container11, 0));
    ApplicationId application2 = BuilderUtils.newApplicationId(1234, 2);
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(application2, 1);
    File app2LogDir = new File(localLogDir, application2.toString());
    app2LogDir.mkdir();
    LogAggregationContext contextWithAMOnly = Records.newRecord(LogAggregationContext.class);
    contextWithAMOnly.setLogAggregationPolicyClassName(AMOnlyLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(application2, this.user, null, this.acls, contextWithAMOnly));
    ContainerId container21 = createContainer(appAttemptId2, 1, ContainerType.APPLICATION_MASTER);
    writeContainerLogs(app2LogDir, container21, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container21, 0));
    ContainerId container12 = createContainer(appAttemptId1, 2, ContainerType.TASK);
    writeContainerLogs(app1LogDir, container12, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container12, 0));
    ApplicationId application3 = BuilderUtils.newApplicationId(1234, 3);
    ApplicationAttemptId appAttemptId3 = BuilderUtils.newApplicationAttemptId(application3, 1);
    File app3LogDir = new File(localLogDir, application3.toString());
    app3LogDir.mkdir();
    LogAggregationContext contextWithAMAndFailed = Records.newRecord(LogAggregationContext.class);
    contextWithAMAndFailed.setLogAggregationPolicyClassName(AMOrFailedContainerLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(application3, this.user, null, this.acls, contextWithAMAndFailed));
    dispatcher.await();
    ApplicationEvent[] expectedInitEvents = new ApplicationEvent[] { new ApplicationEvent(application1, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED), new ApplicationEvent(application2, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED), new ApplicationEvent(application3, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED) };
    checkEvents(appEventHandler, expectedInitEvents, false, "getType", "getApplicationID");
    reset(appEventHandler);
    ContainerId container31 = createContainer(appAttemptId3, 1, ContainerType.APPLICATION_MASTER);
    writeContainerLogs(app3LogDir, container31, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container31, 0));
    ContainerId container32 = createContainer(appAttemptId3, 2, ContainerType.TASK);
    writeContainerLogs(app3LogDir, container32, fileNames);
    logAggregationService.handle(// Failed 
    new LogHandlerContainerFinishedEvent(container32, 1));
    ContainerId container22 = createContainer(appAttemptId2, 2, ContainerType.TASK);
    writeContainerLogs(app2LogDir, container22, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container22, 0));
    ContainerId container33 = createContainer(appAttemptId3, 3, ContainerType.TASK);
    writeContainerLogs(app3LogDir, container33, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container33, 0));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application2));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application3));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application1));
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
    verifyContainerLogs(logAggregationService, application1, new ContainerId[] { container11, container12 }, fileNames, 3, false);
    verifyContainerLogs(logAggregationService, application2, new ContainerId[] { container21 }, fileNames, 3, false);
    verifyContainerLogs(logAggregationService, application3, new ContainerId[] { container31, container32 }, fileNames, 3, false);
    dispatcher.await();
    ApplicationEvent[] expectedFinishedEvents = new ApplicationEvent[] { new ApplicationEvent(application1, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED), new ApplicationEvent(application2, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED), new ApplicationEvent(application3, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED) };
    checkEvents(appEventHandler, expectedFinishedEvents, false, "getType", "getApplicationID");
}

Example 18 with LogAggregationContext

use of org.apache.hadoop.yarn.api.records.LogAggregationContext in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationService.

@SuppressWarnings("unchecked")
private void testLogAggregationService(boolean retentionSizeLimitation) throws Exception {
    LogAggregationContext logAggregationContextWithInterval = Records.newRecord(LogAggregationContext.class);
    // set IncludePattern/excludePattern in rolling fashion
    // we expect all the logs except std_final will be uploaded
    // when app is running. The std_final will be uploaded when
    // the app finishes.
    logAggregationContextWithInterval.setRolledLogsIncludePattern(".*");
    logAggregationContextWithInterval.setRolledLogsExcludePattern("std_final");
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    //configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to
    //have fully qualified path
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.toURI().toString());
    this.conf.setLong(YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, 3600);
    if (retentionSizeLimitation) {
        // set the retention size as 1. The number of logs for one application
        // in one NM should be 1.
        this.conf.setInt(YarnConfiguration.NM_PREFIX + "log-aggregation.num-log-files-per-app", 1);
    }
    // by setting this configuration, the log files will not be deleted immediately after
    // they are aggregated to remote directory.
    // We could use it to test whether the previous aggregated log files will be aggregated
    // again in next cycle.
    this.conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600);
    ApplicationId application = BuilderUtils.newApplicationId(System.currentTimeMillis(), 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(application, 1);
    ContainerId container = createContainer(appAttemptId, 1, ContainerType.APPLICATION_MASTER);
    ConcurrentMap<ApplicationId, Application> maps = this.context.getApplications();
    Application app = mock(Application.class);
    maps.put(application, app);
    when(app.getContainers()).thenReturn(this.context.getContainers());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    // AppLogDir should be created
    File appLogDir = new File(localLogDir, application.toString());
    appLogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application, this.user, null, this.acls, logAggregationContextWithInterval));
    LogFileStatusInLastCycle logFileStatusInLastCycle = null;
    // Simulate log-file creation
    // create std_final in log directory which will not be aggregated
    // until the app finishes.
    String[] logFiles1WithFinalLog = new String[] { "stdout", "stderr", "syslog", "std_final" };
    String[] logFiles1 = new String[] { "stdout", "stderr", "syslog" };
    writeContainerLogs(appLogDir, container, logFiles1WithFinalLog);
    // Do log aggregation
    AppLogAggregatorImpl aggregator = (AppLogAggregatorImpl) logAggregationService.getAppLogAggregators().get(application);
    aggregator.doLogAggregationOutOfBand();
    if (retentionSizeLimitation) {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, true, null));
    } else {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, false, null));
    }
    // Container logs should be uploaded
    logFileStatusInLastCycle = verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles1, 3, true);
    for (String logFile : logFiles1) {
        Assert.assertTrue(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains(logFile));
    }
    // Make sure the std_final is not uploaded.
    Assert.assertFalse(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains("std_final"));
    Thread.sleep(2000);
    // There is no log generated at this time. Do the log aggregation again.
    aggregator.doLogAggregationOutOfBand();
    // Same logs will not be aggregated again.
    // Only one aggregated log file in Remote file directory.
    Assert.assertTrue("Only one aggregated log file in Remote file directory expected", waitAndCheckLogNum(logAggregationService, application, 50, 1, true, null));
    Thread.sleep(2000);
    // Do log aggregation
    String[] logFiles2 = new String[] { "stdout_1", "stderr_1", "syslog_1" };
    writeContainerLogs(appLogDir, container, logFiles2);
    aggregator.doLogAggregationOutOfBand();
    if (retentionSizeLimitation) {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, true, logFileStatusInLastCycle.getLogFilePathInLastCycle()));
    } else {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 2, false, null));
    }
    // Container logs should be uploaded
    logFileStatusInLastCycle = verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles2, 3, true);
    for (String logFile : logFiles2) {
        Assert.assertTrue(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains(logFile));
    }
    // Make sure the std_final is not uploaded.
    Assert.assertFalse(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains("std_final"));
    Thread.sleep(2000);
    // create another logs
    String[] logFiles3 = new String[] { "stdout_2", "stderr_2", "syslog_2" };
    writeContainerLogs(appLogDir, container, logFiles3);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container, 0));
    dispatcher.await();
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application));
    if (retentionSizeLimitation) {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, true, logFileStatusInLastCycle.getLogFilePathInLastCycle()));
    } else {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 3, false, null));
    }
    // the app is finished. The log "std_final" should be aggregated this time.
    String[] logFiles3WithFinalLog = new String[] { "stdout_2", "stderr_2", "syslog_2", "std_final" };
    verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles3WithFinalLog, 4, true);
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
}

Example 19 with LogAggregationContext

use of org.apache.hadoop.yarn.api.records.LogAggregationContext in project hadoop by apache.

the class TestLogAggregationService method testSkipUnnecessaryNNOperationsForShortJob.

@Test(timeout = 20000)
public void testSkipUnnecessaryNNOperationsForShortJob() throws Exception {
    LogAggregationContext logAggregationContext = Records.newRecord(LogAggregationContext.class);
    logAggregationContext.setLogAggregationPolicyClassName(FailedOrKilledContainerLogAggregationPolicy.class.getName());
    verifySkipUnnecessaryNNOperations(logAggregationContext, 0, 2, 0);
}

Example 20 with LogAggregationContext

use of org.apache.hadoop.yarn.api.records.LogAggregationContext in project hadoop by apache.

the class TestLogAggregationService method testSkipUnnecessaryNNOperationsForService.

@Test(timeout = 20000)
public void testSkipUnnecessaryNNOperationsForService() throws Exception {
    this.conf.setLong(YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, 3600);
    LogAggregationContext contextWithAMOnly = Records.newRecord(LogAggregationContext.class);
    contextWithAMOnly.setLogAggregationPolicyClassName(AMOnlyLogAggregationPolicy.class.getName());
    contextWithAMOnly.setRolledLogsIncludePattern("sys*");
    contextWithAMOnly.setRolledLogsExcludePattern("std_final");
    verifySkipUnnecessaryNNOperations(contextWithAMOnly, 1, 4, 1);
}