Examples with ApplicationLocalizationEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent used on opensource projects

Example 16 with ApplicationLocalizationEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent in project hadoop by apache.

the class TestResourceLocalizationService method testResourceRelease.

@Test
// mocked generics
@SuppressWarnings("unchecked")
public void testResourceRelease() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    LocalizerTracker mockLocallilzerTracker = mock(LocalizerTracker.class);
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    //Ignore actual localization
    EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    DeletionService delService = new DeletionService(exec);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker(isA(Configuration.class));
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    try {
        spyService.init(conf);
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES
        LocalResourcesTracker appTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user, appId);
        LocalResourcesTracker privTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId);
        LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // Send localization requests for one resource of each type.
        final LocalResource privResource = getPrivateMockedResource(r);
        final LocalResourceRequest privReq = new LocalResourceRequest(privResource);
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        final LocalResource pubResource2 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2);
        final LocalResource appResource = getAppMockedResource(r);
        final LocalResourceRequest appReq = new LocalResourceRequest(appResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq));
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        req.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq));
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req2.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq));
        req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2));
        Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
        pubRsrcs.add(pubReq);
        pubRsrcs.add(pubReq2);
        // Send Request event
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        spyService.handle(new ContainerLocalizationRequestEvent(c, req2));
        dispatcher.await();
        int privRsrcCount = 0;
        for (LocalizedResource lr : privTracker) {
            privRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 2, lr.getRefCount());
            Assert.assertEquals(privReq, lr.getRequest());
        }
        Assert.assertEquals(1, privRsrcCount);
        int pubRsrcCount = 0;
        for (LocalizedResource lr : pubTracker) {
            pubRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            pubRsrcs.remove(lr.getRequest());
        }
        Assert.assertEquals(0, pubRsrcs.size());
        Assert.assertEquals(2, pubRsrcCount);
        int appRsrcCount = 0;
        for (LocalizedResource lr : appTracker) {
            appRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            Assert.assertEquals(appReq, lr.getRequest());
        }
        Assert.assertEquals(1, appRsrcCount);
        //Send Cleanup Event
        spyService.handle(new ContainerLocalizationCleanupEvent(c, req));
        verify(mockLocallilzerTracker).cleanupPrivLocalizers("container_314159265358979_0003_01_000042");
        req2.remove(LocalResourceVisibility.PRIVATE);
        spyService.handle(new ContainerLocalizationCleanupEvent(c, req2));
        dispatcher.await();
        pubRsrcs.add(pubReq);
        pubRsrcs.add(pubReq2);
        privRsrcCount = 0;
        for (LocalizedResource lr : privTracker) {
            privRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            Assert.assertEquals(privReq, lr.getRequest());
        }
        Assert.assertEquals(1, privRsrcCount);
        pubRsrcCount = 0;
        for (LocalizedResource lr : pubTracker) {
            pubRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 0, lr.getRefCount());
            pubRsrcs.remove(lr.getRequest());
        }
        Assert.assertEquals(0, pubRsrcs.size());
        Assert.assertEquals(2, pubRsrcCount);
        appRsrcCount = 0;
        for (LocalizedResource lr : appTracker) {
            appRsrcCount++;
        }
        Assert.assertEquals(0, appRsrcCount);
    } finally {
        dispatcher.stop();
        delService.stop();
    }
}