Examples with ApplicationEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent used on opensource projects

Example 16 with ApplicationEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent in project hadoop by apache.

the class TestResourceLocalizationService method testLocalizerRunnerException.

@Test(timeout = 10000)
// mocked generics
@SuppressWarnings("unchecked")
public void testLocalizerRunnerException() throws Exception {
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
    dirsHandlerSpy.init(conf);
    DeletionService delServiceReal = new DeletionService(exec);
    DeletionService delService = spy(delServiceReal);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    try {
        spyService.init(conf);
        spyService.start();
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn("user0");
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c = getMockContainer(appId, 42, "user0");
        final LocalResource resource1 = getPrivateMockedResource(r);
        System.out.println("Here 4");
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>();
        privateResourceList.add(req1);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
        final Constructor<?>[] constructors = FSError.class.getDeclaredConstructors();
        constructors[0].setAccessible(true);
        FSError fsError = (FSError) constructors[0].newInstance(new IOException("Disk Error"));
        Mockito.doThrow(fsError).when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class));
        spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
        Thread.sleep(1000);
        dispatcher.await();
        // Verify if ContainerResourceFailedEvent is invoked on FSError
        verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
    } finally {
        spyService.stop();
        dispatcher.stop();
        delService.stop();
    }
}

Example 17 with ApplicationEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent in project hadoop by apache.

the class TestResourceLocalizationService method testFailedPublicResource.

@Test(timeout = 20000)
// mocked generics
@SuppressWarnings("unchecked")
public void testFailedPublicResource() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    DrainDispatcher dispatcher = new DrainDispatcher();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = mock(DeletionService.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    dispatcher.init(conf);
    dispatcher.start();
    try {
        ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
        ResourceLocalizationService spyService = spy(rawService);
        doReturn(mockServer).when(spyService).createServer();
        doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
        spyService.init(conf);
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // cause chmod to fail after a delay
        final CyclicBarrier barrier = new CyclicBarrier(2);
        doAnswer(new Answer<Void>() {

            public Void answer(InvocationOnMock invocation) throws IOException {
                try {
                    barrier.await();
                } catch (InterruptedException e) {
                } catch (BrokenBarrierException e) {
                }
                throw new IOException("forced failure");
            }
        }).when(spylfs).setPermission(isA(Path.class), isA(FsPermission.class));
        // Queue up two localization requests for the same public resource
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
        pubRsrcs.add(pubReq);
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        dispatcher.await();
        // allow the chmod to fail now that both requests have been queued
        barrier.await();
        verify(containerBus, timeout(5000).times(2)).handle(isA(ContainerResourceFailedEvent.class));
    } finally {
        dispatcher.stop();
    }
}

Example 18 with ApplicationEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent in project hadoop by apache.

the class TestLogAggregationService method finishApplication.

private void finishApplication(ApplicationId appId, LogAggregationService logAggregationService) throws Exception {
    dispatcher.await();
    ApplicationEvent[] expectedInitEvents = new ApplicationEvent[] { new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_INITED) };
    checkEvents(appEventHandler, expectedInitEvents, false, "getType", "getApplicationID");
    reset(new EventHandler[] { appEventHandler });
    logAggregationService.handle(new LogHandlerAppFinishedEvent(appId));
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
}

Example 19 with ApplicationEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationCreateDirsFailsWithoutKillingNM.

@Test
public void testLogAggregationCreateDirsFailsWithoutKillingNM() throws Exception {
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    DeletionService spyDelSrvc = spy(this.delSrvc);
    LogAggregationService logAggregationService = spy(new LogAggregationService(dispatcher, this.context, spyDelSrvc, super.dirsHandler));
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId appId = BuilderUtils.newApplicationId(System.currentTimeMillis(), (int) (Math.random() * 1000));
    File appLogDir = new File(localLogDir, appId.toString());
    appLogDir.mkdir();
    Exception e = new RuntimeException("KABOOM!");
    doThrow(e).when(logAggregationService).createAppDir(any(String.class), any(ApplicationId.class), any(UserGroupInformation.class));
    LogAggregationContext contextWithAMAndFailed = Records.newRecord(LogAggregationContext.class);
    contextWithAMAndFailed.setLogAggregationPolicyClassName(AMOrFailedContainerLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, contextWithAMAndFailed));
    dispatcher.await();
    ApplicationEvent[] expectedEvents = new ApplicationEvent[] { new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED) };
    checkEvents(appEventHandler, expectedEvents, false, "getType", "getApplicationID", "getDiagnostic");
    // verify trying to collect logs for containers/apps we don't know about
    // doesn't blow up and tear down the NM
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(BuilderUtils.newContainerId(4, 1, 1, 1), 0));
    dispatcher.await();
    logAggregationService.handle(new LogHandlerAppFinishedEvent(BuilderUtils.newApplicationId(1, 5)));
    dispatcher.await();
    logAggregationService.stop();
    assertEquals(0, logAggregationService.getNumAggregators());
    // local log dir shouldn't be deleted given log aggregation cannot
    // continue due to aggregated log dir creation failure on remoteFS.
    verify(spyDelSrvc, never()).delete(eq(user), any(Path.class), Mockito.<Path>anyVararg());
    verify(logAggregationService).closeFileSystems(any(UserGroupInformation.class));
    // make sure local log dir is not deleted in case log aggregation
    // service cannot be initiated.
    assertTrue(appLogDir.exists());
}

Example 20 with ApplicationEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent in project hadoop by apache.

the class TestLogAggregationService method testLogAggregationInitAppFailsWithoutKillingNM.

@Test
@SuppressWarnings("unchecked")
public void testLogAggregationInitAppFailsWithoutKillingNM() throws Exception {
    this.conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    this.conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, this.remoteRootLogDir.getAbsolutePath());
    LogAggregationService logAggregationService = spy(new LogAggregationService(dispatcher, this.context, this.delSrvc, super.dirsHandler));
    logAggregationService.init(this.conf);
    logAggregationService.start();
    ApplicationId appId = BuilderUtils.newApplicationId(System.currentTimeMillis(), (int) (Math.random() * 1000));
    doThrow(new YarnRuntimeException("KABOOM!")).when(logAggregationService).initAppAggregator(eq(appId), eq(user), any(Credentials.class), anyMap(), any(LogAggregationContext.class), anyLong());
    LogAggregationContext contextWithAMAndFailed = Records.newRecord(LogAggregationContext.class);
    contextWithAMAndFailed.setLogAggregationPolicyClassName(AMOrFailedContainerLogAggregationPolicy.class.getName());
    logAggregationService.handle(new LogHandlerAppStartedEvent(appId, this.user, null, this.acls, contextWithAMAndFailed));
    dispatcher.await();
    ApplicationEvent[] expectedEvents = new ApplicationEvent[] { new ApplicationEvent(appId, ApplicationEventType.APPLICATION_LOG_HANDLING_FAILED) };
    checkEvents(appEventHandler, expectedEvents, false, "getType", "getApplicationID", "getDiagnostic");
    // no filesystems instantiated yet
    verify(logAggregationService, never()).closeFileSystems(any(UserGroupInformation.class));
    // verify trying to collect logs for containers/apps we don't know about
    // doesn't blow up and tear down the NM
    logAggregationService.handle(new LogHandlerContainerFinishedEvent(BuilderUtils.newContainerId(4, 1, 1, 1), 0));
    dispatcher.await();
    logAggregationService.handle(new LogHandlerAppFinishedEvent(BuilderUtils.newApplicationId(1, 5)));
    dispatcher.await();
}