Examples with LogHandlerAppFinishedEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent used on opensource projects

Example 16 with LogHandlerAppFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationServiceWithPatternsAndIntervals.

@SuppressWarnings("resource")
@Test(timeout = 50000)
public void testLogAggregationServiceWithPatternsAndIntervals() throws Exception {
    LogAggregationContext logAggregationContext = Records.newRecord(LogAggregationContext.class);
    // set IncludePattern and RolledLogsIncludePattern.
    // When the app is running, we only aggregate the log with
    // the name stdout. After the app finishes, we only aggregate
    // the log with the name std_final.
    logAggregationContext.setRolledLogsIncludePattern("stdout");
    logAggregationContext.setIncludePattern("std_final");
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    //configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to
    //have fully qualified path
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.toURI().toString());
    this.conf.setLong(YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, 3600);
    this.conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600);
    ApplicationId application = BuilderUtils.newApplicationId(System.currentTimeMillis(), 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(application, 1);
    ContainerId container = createContainer(appAttemptId, 1, ContainerType.APPLICATION_MASTER);
    ConcurrentMap<ApplicationId, Application> maps = this.context.getApplications();
    Application app = mock(Application.class);
    maps.put(application, app);
    when(app.getContainers()).thenReturn(this.context.getContainers());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    // AppLogDir should be created
    File appLogDir = new File(localLogDir, ConverterUtils.toString(application));
    appLogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application, this.user, null, this.acls, logAggregationContext));
    // Simulate log-file creation
    // create std_final in log directory which will not be aggregated
    // until the app finishes.
    String[] logFilesWithFinalLog = new String[] { "stdout", "std_final" };
    writeContainerLogs(appLogDir, container, logFilesWithFinalLog);
    // Do log aggregation
    AppLogAggregatorImpl aggregator = (AppLogAggregatorImpl) logAggregationService.getAppLogAggregators().get(application);
    aggregator.doLogAggregationOutOfBand();
    Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, false, null));
    String[] logFiles = new String[] { "stdout" };
    verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles, 1, true);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container, 0));
    dispatcher.await();
    // Do the log aggregation after ContainerFinishedEvent but before
    // AppFinishedEvent. The std_final is expected to be aggregated this time
    // even if the app is running but the container finishes.
    aggregator.doLogAggregationOutOfBand();
    Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 2, false, null));
    // This container finishes.
    // The log "std_final" should be aggregated this time.
    String[] logFinalLog = new String[] { "std_final" };
    verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFinalLog, 1, true);
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application));
    logAggregationService.stop();
}

Example 17 with LogHandlerAppFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent in project hadoop by apache.

the class TestLogAggregationService method testMultipleAppsLogAggregation.

@Test
@SuppressWarnings("unchecked")
public void testMultipleAppsLogAggregation() throws Exception {
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    String[] fileNames = new String[] { "stdout", "stderr", "syslog" };
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    // AppLogDir should be created
    File app1LogDir = new File(localLogDir, application1.toString());
    app1LogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application1, this.user, null, this.acls));
    ApplicationAttemptId appAttemptId1 = BuilderUtils.newApplicationAttemptId(application1, 1);
    ContainerId container11 = createContainer(appAttemptId1, 1, ContainerType.APPLICATION_MASTER);
    // Simulate log-file creation
    writeContainerLogs(app1LogDir, container11, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container11, 0));
    ApplicationId application2 = BuilderUtils.newApplicationId(1234, 2);
    ApplicationAttemptId appAttemptId2 = BuilderUtils.newApplicationAttemptId(application2, 1);
    File app2LogDir = new File(localLogDir, application2.toString());
    app2LogDir.mkdir();
    LogAggregationContext contextWithAMOnly = Records.newRecord(LogAggregationContext.class);
    contextWithAMOnly.setLogAggregationPolicyClassName(AMOnlyLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(application2, this.user, null, this.acls, contextWithAMOnly));
    ContainerId container21 = createContainer(appAttemptId2, 1, ContainerType.APPLICATION_MASTER);
    writeContainerLogs(app2LogDir, container21, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container21, 0));
    ContainerId container12 = createContainer(appAttemptId1, 2, ContainerType.TASK);
    writeContainerLogs(app1LogDir, container12, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container12, 0));
    ApplicationId application3 = BuilderUtils.newApplicationId(1234, 3);
    ApplicationAttemptId appAttemptId3 = BuilderUtils.newApplicationAttemptId(application3, 1);
    File app3LogDir = new File(localLogDir, application3.toString());
    app3LogDir.mkdir();
    LogAggregationContext contextWithAMAndFailed = Records.newRecord(LogAggregationContext.class);
    contextWithAMAndFailed.setLogAggregationPolicyClassName(AMOrFailedContainerLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(application3, this.user, null, this.acls, contextWithAMAndFailed));
    dispatcher.await();
    ApplicationEvent[] expectedInitEvents = new ApplicationEvent[] { new ApplicationEvent(application1, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED), new ApplicationEvent(application2, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED), new ApplicationEvent(application3, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED) };
    checkEvents(appEventHandler, expectedInitEvents, false, "getType", "getApplicationID");
    reset(appEventHandler);
    ContainerId container31 = createContainer(appAttemptId3, 1, ContainerType.APPLICATION_MASTER);
    writeContainerLogs(app3LogDir, container31, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container31, 0));
    ContainerId container32 = createContainer(appAttemptId3, 2, ContainerType.TASK);
    writeContainerLogs(app3LogDir, container32, fileNames);
    logAggregationService.handle(// Failed 
    new LogHandlerContainerFinishedEvent(container32, 1));
    ContainerId container22 = createContainer(appAttemptId2, 2, ContainerType.TASK);
    writeContainerLogs(app2LogDir, container22, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container22, 0));
    ContainerId container33 = createContainer(appAttemptId3, 3, ContainerType.TASK);
    writeContainerLogs(app3LogDir, container33, fileNames);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container33, 0));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application2));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application3));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application1));
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
    verifyContainerLogs(logAggregationService, application1, new ContainerId[] { container11, container12 }, fileNames, 3, false);
    verifyContainerLogs(logAggregationService, application2, new ContainerId[] { container21 }, fileNames, 3, false);
    verifyContainerLogs(logAggregationService, application3, new ContainerId[] { container31, container32 }, fileNames, 3, false);
    dispatcher.await();
    ApplicationEvent[] expectedFinishedEvents = new ApplicationEvent[] { new ApplicationEvent(application1, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED), new ApplicationEvent(application2, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED), new ApplicationEvent(application3, ApplicationEventType.APPLICATION_LOG_HANDLING_FINISHED) };
    checkEvents(appEventHandler, expectedFinishedEvents, false, "getType", "getApplicationID");
}

Example 18 with LogHandlerAppFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent in project hadoop by apache.

the class TestLogAggregationService method testLogAggregatorCleanup.

@Test
public void testLogAggregatorCleanup() throws Exception {
    DeletionService delSrvc = mock(DeletionService.class);
    // get the AppLogAggregationImpl thread to crash
    LocalDirsHandlerService mockedDirSvc = mock(LocalDirsHandlerService.class);
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, delSrvc, mockedDirSvc);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    logAggregationService.handle(new LogHandlerAppStartedEvent(application1, this.user, null, this.acls));
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application1));
    dispatcher.await();
    int timeToWait = 20 * 1000;
    while (timeToWait > 0 && logAggregationService.getNumAggregators() > 0) {
        Thread.sleep(100);
        timeToWait -= 100;
    }
    Assert.assertEquals("Log aggregator failed to cleanup!", 0, logAggregationService.getNumAggregators());
    logAggregationService.stop();
    logAggregationService.close();
}

Example 19 with LogHandlerAppFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent in project hadoop by apache.

the class TestLogAggregationService method testAddNewTokenSentFromRMForLogAggregation.

@Test(timeout = 20000)
public void testAddNewTokenSentFromRMForLogAggregation() throws Exception {
    Configuration conf = new YarnConfiguration();
    conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
    UserGroupInformation.setConfiguration(conf);
    ApplicationId application1 = BuilderUtils.newApplicationId(1234, 1);
    Application mockApp = mock(Application.class);
    when(mockApp.getContainers()).thenReturn(new HashMap<ContainerId, Container>());
    this.context.getApplications().put(application1, mockApp);
    @SuppressWarnings("resource") LogAggregationService logAggregationService = new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application1, this.user, null, this.acls, Records.newRecord(LogAggregationContext.class)));
    // Inject new token for log-aggregation after app log-aggregator init
    Text userText1 = new Text("user1");
    RMDelegationTokenIdentifier dtId1 = new RMDelegationTokenIdentifier(userText1, new Text("renewer1"), userText1);
    final Token<RMDelegationTokenIdentifier> token1 = new Token<RMDelegationTokenIdentifier>(dtId1.getBytes(), "password1".getBytes(), dtId1.getKind(), new Text("service1"));
    Credentials credentials = new Credentials();
    credentials.addToken(userText1, token1);
    this.context.getSystemCredentialsForApps().put(application1, credentials);
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application1));
    final UserGroupInformation ugi = ((AppLogAggregatorImpl) logAggregationService.getAppLogAggregators().get(application1)).getUgi();
    GenericTestUtils.waitFor(new Supplier<Boolean>() {

        public Boolean get() {
            boolean hasNewToken = false;
            for (Token<?> token : ugi.getCredentials().getAllTokens()) {
                if (token.equals(token1)) {
                    hasNewToken = true;
                }
            }
            return hasNewToken;
        }
    }, 1000, 20000);
    logAggregationService.stop();
}

Example 20 with LogHandlerAppFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationService.

@SuppressWarnings("unchecked")
private void testLogAggregationService(boolean retentionSizeLimitation) throws Exception {
    LogAggregationContext logAggregationContextWithInterval = Records.newRecord(LogAggregationContext.class);
    // set IncludePattern/excludePattern in rolling fashion
    // we expect all the logs except std_final will be uploaded
    // when app is running. The std_final will be uploaded when
    // the app finishes.
    logAggregationContextWithInterval.setRolledLogsIncludePattern(".*");
    logAggregationContextWithInterval.setRolledLogsExcludePattern("std_final");
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    //configure YarnConfiguration.NM_REMOTE_APP_LOG_DIR to
    //have fully qualified path
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.toURI().toString());
    this.conf.setLong(YarnConfiguration.NM_LOG_AGGREGATION_ROLL_MONITORING_INTERVAL_SECONDS, 3600);
    if (retentionSizeLimitation) {
        // set the retention size as 1. The number of logs for one application
        // in one NM should be 1.
        this.conf.setInt(YarnConfiguration.NM_PREFIX + "log-aggregation.num-log-files-per-app", 1);
    }
    // by setting this configuration, the log files will not be deleted immediately after
    // they are aggregated to remote directory.
    // We could use it to test whether the previous aggregated log files will be aggregated
    // again in next cycle.
    this.conf.setLong(YarnConfiguration.DEBUG_NM_DELETE_DELAY_SEC, 3600);
    ApplicationId application = BuilderUtils.newApplicationId(System.currentTimeMillis(), 1);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(application, 1);
    ContainerId container = createContainer(appAttemptId, 1, ContainerType.APPLICATION_MASTER);
    ConcurrentMap<ApplicationId, Application> maps = this.context.getApplications();
    Application app = mock(Application.class);
    maps.put(application, app);
    when(app.getContainers()).thenReturn(this.context.getContainers());
    LogAggregationService logAggregationService = new LogAggregationService(dispatcher, context, this.delSrvc, super.dirsHandler);
    logAggregationService.init(this.conf);
    logAggregationService.start();
    // AppLogDir should be created
    File appLogDir = new File(localLogDir, application.toString());
    appLogDir.mkdir();
    logAggregationService.handle(new LogHandlerAppStartedEvent(application, this.user, null, this.acls, logAggregationContextWithInterval));
    LogFileStatusInLastCycle logFileStatusInLastCycle = null;
    // Simulate log-file creation
    // create std_final in log directory which will not be aggregated
    // until the app finishes.
    String[] logFiles1WithFinalLog = new String[] { "stdout", "stderr", "syslog", "std_final" };
    String[] logFiles1 = new String[] { "stdout", "stderr", "syslog" };
    writeContainerLogs(appLogDir, container, logFiles1WithFinalLog);
    // Do log aggregation
    AppLogAggregatorImpl aggregator = (AppLogAggregatorImpl) logAggregationService.getAppLogAggregators().get(application);
    aggregator.doLogAggregationOutOfBand();
    if (retentionSizeLimitation) {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, true, null));
    } else {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, false, null));
    }
    // Container logs should be uploaded
    logFileStatusInLastCycle = verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles1, 3, true);
    for (String logFile : logFiles1) {
        Assert.assertTrue(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains(logFile));
    }
    // Make sure the std_final is not uploaded.
    Assert.assertFalse(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains("std_final"));
    Thread.sleep(2000);
    // There is no log generated at this time. Do the log aggregation again.
    aggregator.doLogAggregationOutOfBand();
    // Same logs will not be aggregated again.
    // Only one aggregated log file in Remote file directory.
    Assert.assertTrue("Only one aggregated log file in Remote file directory expected", waitAndCheckLogNum(logAggregationService, application, 50, 1, true, null));
    Thread.sleep(2000);
    // Do log aggregation
    String[] logFiles2 = new String[] { "stdout_1", "stderr_1", "syslog_1" };
    writeContainerLogs(appLogDir, container, logFiles2);
    aggregator.doLogAggregationOutOfBand();
    if (retentionSizeLimitation) {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, true, logFileStatusInLastCycle.getLogFilePathInLastCycle()));
    } else {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 2, false, null));
    }
    // Container logs should be uploaded
    logFileStatusInLastCycle = verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles2, 3, true);
    for (String logFile : logFiles2) {
        Assert.assertTrue(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains(logFile));
    }
    // Make sure the std_final is not uploaded.
    Assert.assertFalse(logFileStatusInLastCycle.getLogFileTypesInLastCycle().contains("std_final"));
    Thread.sleep(2000);
    // create another logs
    String[] logFiles3 = new String[] { "stdout_2", "stderr_2", "syslog_2" };
    writeContainerLogs(appLogDir, container, logFiles3);
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(container, 0));
    dispatcher.await();
    logAggregationService.handle(new LogHandlerAppFinishedEvent(application));
    if (retentionSizeLimitation) {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 1, true, logFileStatusInLastCycle.getLogFilePathInLastCycle()));
    } else {
        Assert.assertTrue(waitAndCheckLogNum(logAggregationService, application, 50, 3, false, null));
    }
    // the app is finished. The log "std_final" should be aggregated this time.
    String[] logFiles3WithFinalLog = new String[] { "stdout_2", "stderr_2", "syslog_2", "std_final" };
    verifyContainerLogs(logAggregationService, application, new ContainerId[] { container }, logFiles3WithFinalLog, 4, true);
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
}