Examples with ApplicationEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent used on opensource projects

Example 6 with ApplicationEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent in project hadoop by apache.

the class AppLogAggregatorImpl method run.

@SuppressWarnings("unchecked")
@Override
public void run() {
    try {
        doAppLogAggregation();
    } catch (Exception e) {
        // do post clean up of log directories on any exception
        LOG.error("Error occurred while aggregating the log for the application " + appId, e);
        doAppLogAggregationPostCleanUp();
    } finally {
        if (!this.appAggregationFinished.get() && !this.aborted.get()) {
            LOG.warn("Log aggregation did not complete for application " + appId);
            this.dispatcher.getEventHandler().handle(new ApplicationEvent(this.appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED));
        }
        this.appAggregationFinished.set(true);
    }
}

Example 7 with ApplicationEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent in project hadoop by apache.

the class LogAggregationService method stopApp.

@SuppressWarnings("unchecked")
private void stopApp(ApplicationId appId) {
    // App is complete. Finish up any containers' pending log aggregation and
    // close the application specific logFile.
    AppLogAggregator aggregator = this.appLogAggregators.get(appId);
    if (aggregator == null) {
        LOG.warn("Log aggregation is not initialized for " + appId + ", did it fail to start?");
        this.dispatcher.getEventHandler().handle(new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED));
        return;
    }
    aggregator.finishLogAggregation();
}

Example 8 with ApplicationEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent in project hadoop by apache.

the class NonAggregatingLogHandler method handle.

@SuppressWarnings("unchecked")
@Override
public void handle(LogHandlerEvent event) {
    switch(event.getType()) {
        case APPLICATION_STARTED:
            LogHandlerAppStartedEvent appStartedEvent = (LogHandlerAppStartedEvent) event;
            this.appOwners.put(appStartedEvent.getApplicationId(), appStartedEvent.getUser());
            this.dispatcher.getEventHandler().handle(new ApplicationEvent(appStartedEvent.getApplicationId(), ApplicationEventType.APPLICATION_LOG_HANDLING_INITED));
            break;
        case CONTAINER_FINISHED:
            // Ignore
            break;
        case APPLICATION_FINISHED:
            LogHandlerAppFinishedEvent appFinishedEvent = (LogHandlerAppFinishedEvent) event;
            ApplicationId appId = appFinishedEvent.getApplicationId();
            // Schedule - so that logs are available on the UI till they're deleted.
            LOG.info("Scheduling Log Deletion for application: " + appId + ", with delay of " + this.deleteDelaySeconds + " seconds");
            String user = appOwners.remove(appId);
            if (user == null) {
                LOG.error("Unable to locate user for " + appId);
                // send LOG_HANDLING_FAILED out
                NonAggregatingLogHandler.this.dispatcher.getEventHandler().handle(new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED));
                break;
            }
            LogDeleterRunnable logDeleter = new LogDeleterRunnable(user, appId);
            long deletionTimestamp = System.currentTimeMillis() + this.deleteDelaySeconds * 1000;
            LogDeleterProto deleterProto = LogDeleterProto.newBuilder().setUser(user).setDeletionTime(deletionTimestamp).build();
            try {
                stateStore.storeLogDeleter(appId, deleterProto);
            } catch (IOException e) {
                LOG.error("Unable to record log deleter state", e);
            }
            try {
                sched.schedule(logDeleter, this.deleteDelaySeconds, TimeUnit.SECONDS);
            } catch (RejectedExecutionException e) {
                // Handling this event in local thread before starting threads
                // or after calling sched.shutdownNow().
                logDeleter.run();
            }
            break;
        default:
            // Ignore
            ;
    }
}

Example 9 with ApplicationEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent in project hadoop by apache.

the class DummyContainerManager method createResourceLocalizationService.

@Override
@SuppressWarnings("unchecked")
protected ResourceLocalizationService createResourceLocalizationService(ContainerExecutor exec, DeletionService deletionContext, Context context) {
    return new ResourceLocalizationService(super.dispatcher, exec, deletionContext, super.dirsHandler, context) {

        @Override
        public void handle(LocalizationEvent event) {
            switch(event.getType()) {
                case INIT_APPLICATION_RESOURCES:
                    Application app = ((ApplicationLocalizationEvent) event).getApplication();
                    // Simulate event from ApplicationLocalization.
                    dispatcher.getEventHandler().handle(new ApplicationInitedEvent(app.getAppId()));
                    break;
                case LOCALIZE_CONTAINER_RESOURCES:
                    ContainerLocalizationRequestEvent rsrcReqs = (ContainerLocalizationRequestEvent) event;
                    // simulate localization of all requested resources
                    for (Collection<LocalResourceRequest> rc : rsrcReqs.getRequestedResources().values()) {
                        for (LocalResourceRequest req : rc) {
                            LOG.info("DEBUG: " + req + ":" + rsrcReqs.getContainer().getContainerId());
                            dispatcher.getEventHandler().handle(new ContainerResourceLocalizedEvent(rsrcReqs.getContainer().getContainerId(), req, new Path("file:///local" + req.getPath().toUri().getPath())));
                        }
                    }
                    break;
                case CLEANUP_CONTAINER_RESOURCES:
                    Container container = ((ContainerLocalizationEvent) event).getContainer();
                    // TODO: delete the container dir
                    this.dispatcher.getEventHandler().handle(new ContainerEvent(container.getContainerId(), ContainerEventType.CONTAINER_RESOURCES_CLEANEDUP));
                    break;
                case DESTROY_APPLICATION_RESOURCES:
                    Application application = ((ApplicationLocalizationEvent) event).getApplication();
                    // decrement reference counts of all resources associated with this
                    // app
                    this.dispatcher.getEventHandler().handle(new ApplicationEvent(application.getAppId(), ApplicationEventType.APPLICATION_RESOURCES_CLEANEDUP));
                    break;
                default:
                    fail("Unexpected event: " + event.getType());
            }
        }
    };
}

Example 10 with ApplicationEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent in project hadoop by apache.

the class TestResourceLocalizationService method testPublicResourceInitializesLocalDir.

@Test
@SuppressWarnings("unchecked")
public void testPublicResourceInitializesLocalDir() throws Exception {
    // Setup state to simulate restart NM with existing state meaning no
    // directory creation during initialization
    NMStateStoreService spyStateStore = spy(nmContext.getNMStateStore());
    when(spyStateStore.canRecover()).thenReturn(true);
    NMContext spyContext = spy(nmContext);
    when(spyContext.getNMStateStore()).thenReturn(spyStateStore);
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    DrainDispatcher dispatcher = new DrainDispatcher();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = mock(DeletionService.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    dispatcher.init(conf);
    dispatcher.start();
    try {
        ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, spyContext);
        ResourceLocalizationService spyService = spy(rawService);
        doReturn(mockServer).when(spyService).createServer();
        doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
        spyService.init(conf);
        final FsPermission defaultPerm = new FsPermission((short) 0755);
        // verify directory is not created at initialization
        for (Path p : localDirs) {
            p = new Path((new URI(p.toString())).getPath());
            Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
            verify(spylfs, never()).mkdir(eq(publicCache), eq(defaultPerm), eq(true));
        }
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // Queue up public resource localization
        final LocalResource pubResource1 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq1 = new LocalResourceRequest(pubResource1);
        LocalResource pubResource2 = null;
        do {
            pubResource2 = getPublicMockedResource(r);
        } while (pubResource2 == null || pubResource2.equals(pubResource1));
        // above call to make sure we don't get identical resources.
        final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2);
        Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
        pubRsrcs.add(pubReq1);
        pubRsrcs.add(pubReq2);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PUBLIC, pubRsrcs);
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        dispatcher.await();
        verify(spyService, times(1)).checkAndInitializeLocalDirs();
        // verify directory creation
        for (Path p : localDirs) {
            p = new Path((new URI(p.toString())).getPath());
            Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
            verify(spylfs).mkdir(eq(publicCache), eq(defaultPerm), eq(true));
        }
    } finally {
        dispatcher.stop();
    }
}