Search in sources :

Example 1 with LogHandlerAppFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent in project hadoop by apache.

the class LogAggregationService method handle.

@Override
public void handle(LogHandlerEvent event) {
    switch(event.getType()) {
        case APPLICATION_STARTED:
            LogHandlerAppStartedEvent appStartEvent = (LogHandlerAppStartedEvent) event;
            initApp(appStartEvent.getApplicationId(), appStartEvent.getUser(), appStartEvent.getCredentials(), appStartEvent.getApplicationAcls(), appStartEvent.getLogAggregationContext(), appStartEvent.getRecoveredAppLogInitedTime());
            break;
        case CONTAINER_FINISHED:
            LogHandlerContainerFinishedEvent containerFinishEvent = (LogHandlerContainerFinishedEvent) event;
            stopContainer(containerFinishEvent.getContainerId(), containerFinishEvent.getExitCode());
            break;
        case APPLICATION_FINISHED:
            LogHandlerAppFinishedEvent appFinishedEvent = (LogHandlerAppFinishedEvent) event;
            stopApp(appFinishedEvent.getApplicationId());
            break;
        default:
            // Ignore
            ;
    }
}
Also used : LogHandlerAppStartedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent) LogHandlerAppFinishedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent) LogHandlerContainerFinishedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent)

Example 2 with LogHandlerAppFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent in project hadoop by apache.

the class NonAggregatingLogHandler method handle.

@SuppressWarnings("unchecked")
@Override
public void handle(LogHandlerEvent event) {
    switch(event.getType()) {
        case APPLICATION_STARTED:
            LogHandlerAppStartedEvent appStartedEvent = (LogHandlerAppStartedEvent) event;
            this.appOwners.put(appStartedEvent.getApplicationId(), appStartedEvent.getUser());
            this.dispatcher.getEventHandler().handle(new ApplicationEvent(appStartedEvent.getApplicationId(), ApplicationEventType.APPLICATION_LOG_HANDLING_INITED));
            break;
        case CONTAINER_FINISHED:
            // Ignore
            break;
        case APPLICATION_FINISHED:
            LogHandlerAppFinishedEvent appFinishedEvent = (LogHandlerAppFinishedEvent) event;
            ApplicationId appId = appFinishedEvent.getApplicationId();
            // Schedule - so that logs are available on the UI till they're deleted.
            LOG.info("Scheduling Log Deletion for application: " + appId + ", with delay of " + this.deleteDelaySeconds + " seconds");
            String user = appOwners.remove(appId);
            if (user == null) {
                LOG.error("Unable to locate user for " + appId);
                // send LOG_HANDLING_FAILED out
                NonAggregatingLogHandler.this.dispatcher.getEventHandler().handle(new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED));
                break;
            }
            LogDeleterRunnable logDeleter = new LogDeleterRunnable(user, appId);
            long deletionTimestamp = System.currentTimeMillis() + this.deleteDelaySeconds * 1000;
            LogDeleterProto deleterProto = LogDeleterProto.newBuilder().setUser(user).setDeletionTime(deletionTimestamp).build();
            try {
                stateStore.storeLogDeleter(appId, deleterProto);
            } catch (IOException e) {
                LOG.error("Unable to record log deleter state", e);
            }
            try {
                sched.schedule(logDeleter, this.deleteDelaySeconds, TimeUnit.SECONDS);
            } catch (RejectedExecutionException e) {
                // Handling this event in local thread before starting threads
                // or after calling sched.shutdownNow().
                logDeleter.run();
            }
            break;
        default:
            // Ignore
            ;
    }
}
Also used : LogHandlerAppStartedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent) ApplicationEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent) LogHandlerAppFinishedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent) LogDeleterProto(org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.LogDeleterProto) IOException(java.io.IOException) ApplicationId(org.apache.hadoop.yarn.api.records.ApplicationId) RejectedExecutionException(java.util.concurrent.RejectedExecutionException)

Example 3 with LogHandlerAppFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent in project hadoop by apache.

the class TestNonAggregatingLogHandler method testDelayedDelete.

@Test
public void testDelayedDelete() throws IOException {
    File[] localLogDirs = getLocalLogDirFiles(this.getClass().getName(), 2);
    String localLogDirsString = localLogDirs[0].getAbsolutePath() + "," + localLogDirs[1].getAbsolutePath();
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);
    conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, YarnConfiguration.DEFAULT_NM_LOG_RETAIN_SECONDS);
    dirsHandler.init(conf);
    NonAggregatingLogHandler logHandler = new NonAggregatingLogHandlerWithMockExecutor(dispatcher, mockDelService, dirsHandler);
    logHandler.init(conf);
    logHandler.start();
    logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null, null));
    logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));
    logHandler.handle(new LogHandlerAppFinishedEvent(appId));
    Path[] localAppLogDirs = new Path[2];
    localAppLogDirs[0] = new Path(localLogDirs[0].getAbsolutePath(), appId.toString());
    localAppLogDirs[1] = new Path(localLogDirs[1].getAbsolutePath(), appId.toString());
    ScheduledThreadPoolExecutor mockSched = ((NonAggregatingLogHandlerWithMockExecutor) logHandler).mockSched;
    verify(mockSched).schedule(any(Runnable.class), eq(10800l), eq(TimeUnit.SECONDS));
    logHandler.close();
    for (int i = 0; i < localLogDirs.length; i++) {
        FileUtils.deleteDirectory(localLogDirs[i]);
    }
}
Also used : Path(org.apache.hadoop.fs.Path) ScheduledThreadPoolExecutor(java.util.concurrent.ScheduledThreadPoolExecutor) LogHandlerContainerFinishedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent) LogHandlerAppStartedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent) LogHandlerAppFinishedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent) File(java.io.File) Test(org.junit.Test)

Example 4 with LogHandlerAppFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent in project hadoop by apache.

the class TestNonAggregatingLogHandler method testLogDeletion.

@Test
public void testLogDeletion() throws IOException {
    File[] localLogDirs = getLocalLogDirFiles(this.getClass().getName(), 2);
    String localLogDirsString = localLogDirs[0].getAbsolutePath() + "," + localLogDirs[1].getAbsolutePath();
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDirsString);
    conf.setBoolean(YarnConfiguration.LOG_AGGREGATION_ENABLED, false);
    conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 0l);
    dirsHandler.init(conf);
    NonAggregatingLogHandler rawLogHandler = new NonAggregatingLogHandler(dispatcher, mockDelService, dirsHandler, new NMNullStateStoreService());
    NonAggregatingLogHandler logHandler = spy(rawLogHandler);
    AbstractFileSystem spylfs = spy(FileContext.getLocalFSFileContext().getDefaultFileSystem());
    FileContext lfs = FileContext.getFileContext(spylfs, conf);
    doReturn(lfs).when(logHandler).getLocalFileContext(isA(Configuration.class));
    FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask());
    final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(localLogDirs[0].getAbsolutePath()));
    doReturn(fs).when(spylfs).getFileStatus(isA(Path.class));
    logHandler.init(conf);
    logHandler.start();
    logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null, null));
    logHandler.handle(new LogHandlerContainerFinishedEvent(container11, 0));
    logHandler.handle(new LogHandlerAppFinishedEvent(appId));
    Path[] localAppLogDirs = new Path[2];
    localAppLogDirs[0] = new Path(localLogDirs[0].getAbsolutePath(), appId.toString());
    localAppLogDirs[1] = new Path(localLogDirs[1].getAbsolutePath(), appId.toString());
    testDeletionServiceCall(mockDelService, user, 5000, localAppLogDirs);
    logHandler.close();
    for (int i = 0; i < localLogDirs.length; i++) {
        FileUtils.deleteDirectory(localLogDirs[i]);
    }
}
Also used : Path(org.apache.hadoop.fs.Path) FileStatus(org.apache.hadoop.fs.FileStatus) Configuration(org.apache.hadoop.conf.Configuration) YarnConfiguration(org.apache.hadoop.yarn.conf.YarnConfiguration) LogHandlerContainerFinishedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent) NMNullStateStoreService(org.apache.hadoop.yarn.server.nodemanager.recovery.NMNullStateStoreService) AbstractFileSystem(org.apache.hadoop.fs.AbstractFileSystem) LogHandlerAppStartedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent) LogHandlerAppFinishedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent) FsPermission(org.apache.hadoop.fs.permission.FsPermission) File(java.io.File) FileContext(org.apache.hadoop.fs.FileContext) Test(org.junit.Test)

Example 5 with LogHandlerAppFinishedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent in project hadoop by apache.

the class TestNonAggregatingLogHandler method runMockedFailedDirs.

/**
   * Function to run a log handler with directories failing the getFileStatus
   * call. The function accepts the log handler, setup the mocks to fail with
   * specific exceptions and ensures the deletion service has the correct calls.
   * 
   * @param logHandler the logHandler implementation to test
   * 
   * @param appId the application id that we wish when sending events to the log
   * handler
   * 
   * @param user the user name to use
   * 
   * @param mockDelService a mock of the DeletionService which we will verify
   * the delete calls against
   * 
   * @param dirsHandler a spy or mock on the LocalDirsHandler service used to
   * when creating the logHandler. It needs to be a spy so that we can intercept
   * the getAllLogDirs() call.
   * 
   * @param conf the configuration used
   * 
   * @param spylfs a spy on the AbstractFileSystem object used when creating lfs
   * 
   * @param lfs the FileContext object to be used to mock the getFileStatus()
   * calls
   * 
   * @param localLogDirs list of the log dirs to run the test against, must have
   * at least 7 entries
   */
public static void runMockedFailedDirs(LogHandler logHandler, ApplicationId appId, String user, DeletionService mockDelService, LocalDirsHandlerService dirsHandler, Configuration conf, AbstractFileSystem spylfs, FileContext lfs, File[] localLogDirs) throws Exception {
    Map<ApplicationAccessType, String> appAcls = new HashMap<ApplicationAccessType, String>();
    if (localLogDirs.length < 7) {
        throw new IllegalArgumentException("Argument localLogDirs must be at least of length 7");
    }
    Path[] localAppLogDirPaths = new Path[localLogDirs.length];
    for (int i = 0; i < localAppLogDirPaths.length; i++) {
        localAppLogDirPaths[i] = new Path(localLogDirs[i].getAbsolutePath(), appId.toString());
    }
    final List<String> localLogDirPaths = new ArrayList<String>(localLogDirs.length);
    for (int i = 0; i < localLogDirs.length; i++) {
        localLogDirPaths.add(localLogDirs[i].getAbsolutePath());
    }
    // setup mocks
    FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask());
    final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(localLogDirs[0].getAbsolutePath()));
    doReturn(fs).when(spylfs).getFileStatus(isA(Path.class));
    doReturn(localLogDirPaths).when(dirsHandler).getLogDirsForCleanup();
    logHandler.handle(new LogHandlerAppStartedEvent(appId, user, null, appAcls));
    // test case where some dirs have the log dir to delete
    // mock some dirs throwing various exceptions
    // verify deletion happens only on the others
    Mockito.doThrow(new FileNotFoundException()).when(spylfs).getFileStatus(eq(localAppLogDirPaths[0]));
    doReturn(fs).when(spylfs).getFileStatus(eq(localAppLogDirPaths[1]));
    Mockito.doThrow(new AccessControlException()).when(spylfs).getFileStatus(eq(localAppLogDirPaths[2]));
    doReturn(fs).when(spylfs).getFileStatus(eq(localAppLogDirPaths[3]));
    Mockito.doThrow(new IOException()).when(spylfs).getFileStatus(eq(localAppLogDirPaths[4]));
    Mockito.doThrow(new UnsupportedFileSystemException("test")).when(spylfs).getFileStatus(eq(localAppLogDirPaths[5]));
    doReturn(fs).when(spylfs).getFileStatus(eq(localAppLogDirPaths[6]));
    logHandler.handle(new LogHandlerAppFinishedEvent(appId));
    testDeletionServiceCall(mockDelService, user, 5000, localAppLogDirPaths[1], localAppLogDirPaths[3], localAppLogDirPaths[6]);
    return;
}
Also used : Path(org.apache.hadoop.fs.Path) FileStatus(org.apache.hadoop.fs.FileStatus) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) FileNotFoundException(java.io.FileNotFoundException) AccessControlException(org.apache.hadoop.security.AccessControlException) IOException(java.io.IOException) LogHandlerAppStartedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent) ApplicationAccessType(org.apache.hadoop.yarn.api.records.ApplicationAccessType) LogHandlerAppFinishedEvent(org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent) UnsupportedFileSystemException(org.apache.hadoop.fs.UnsupportedFileSystemException) FsPermission(org.apache.hadoop.fs.permission.FsPermission)

Aggregations

LogHandlerAppFinishedEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent)20 LogHandlerAppStartedEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent)18 ApplicationId (org.apache.hadoop.yarn.api.records.ApplicationId)13 Test (org.junit.Test)13 LogHandlerContainerFinishedEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerContainerFinishedEvent)12 File (java.io.File)11 BaseContainerManagerTest (org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest)9 ApplicationEvent (org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent)8 ContainerId (org.apache.hadoop.yarn.api.records.ContainerId)7 LogAggregationContext (org.apache.hadoop.yarn.api.records.LogAggregationContext)7 ApplicationAttemptId (org.apache.hadoop.yarn.api.records.ApplicationAttemptId)6 Path (org.apache.hadoop.fs.Path)5 UserGroupInformation (org.apache.hadoop.security.UserGroupInformation)4 DeletionService (org.apache.hadoop.yarn.server.nodemanager.DeletionService)4 Application (org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application)4 IOException (java.io.IOException)3 FileNotFoundException (java.io.FileNotFoundException)2 Configuration (org.apache.hadoop.conf.Configuration)2 FileStatus (org.apache.hadoop.fs.FileStatus)2 UnsupportedFileSystemException (org.apache.hadoop.fs.UnsupportedFileSystemException)2