Examples with DrainDispatcher org.apache.hadoop.yarn.event.DrainDispatcher used on opensource projects

Example 21 with DrainDispatcher

use of org.apache.hadoop.yarn.event.DrainDispatcher in project hadoop by apache.

the class TestResourceLocalizationService method testLocalizerRunnerException.

@Test(timeout = 10000)
// mocked generics
@SuppressWarnings("unchecked")
public void testLocalizerRunnerException() throws Exception {
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
    dirsHandlerSpy.init(conf);
    DeletionService delServiceReal = new DeletionService(exec);
    DeletionService delService = spy(delServiceReal);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    try {
        spyService.init(conf);
        spyService.start();
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn("user0");
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c = getMockContainer(appId, 42, "user0");
        final LocalResource resource1 = getPrivateMockedResource(r);
        System.out.println("Here 4");
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>();
        privateResourceList.add(req1);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
        final Constructor<?>[] constructors = FSError.class.getDeclaredConstructors();
        constructors[0].setAccessible(true);
        FSError fsError = (FSError) constructors[0].newInstance(new IOException("Disk Error"));
        Mockito.doThrow(fsError).when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class));
        spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
        Thread.sleep(1000);
        dispatcher.await();
        // Verify if ContainerResourceFailedEvent is invoked on FSError
        verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
    } finally {
        spyService.stop();
        dispatcher.stop();
        delService.stop();
    }
}

Example 22 with DrainDispatcher

use of org.apache.hadoop.yarn.event.DrainDispatcher in project hadoop by apache.

the class TestResourceLocalizationService method testFailedPublicResource.

@Test(timeout = 20000)
// mocked generics
@SuppressWarnings("unchecked")
public void testFailedPublicResource() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    DrainDispatcher dispatcher = new DrainDispatcher();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = mock(DeletionService.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    dispatcher.init(conf);
    dispatcher.start();
    try {
        ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
        ResourceLocalizationService spyService = spy(rawService);
        doReturn(mockServer).when(spyService).createServer();
        doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
        spyService.init(conf);
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // cause chmod to fail after a delay
        final CyclicBarrier barrier = new CyclicBarrier(2);
        doAnswer(new Answer<Void>() {

            public Void answer(InvocationOnMock invocation) throws IOException {
                try {
                    barrier.await();
                } catch (InterruptedException e) {
                } catch (BrokenBarrierException e) {
                }
                throw new IOException("forced failure");
            }
        }).when(spylfs).setPermission(isA(Path.class), isA(FsPermission.class));
        // Queue up two localization requests for the same public resource
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
        pubRsrcs.add(pubReq);
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        dispatcher.await();
        // allow the chmod to fail now that both requests have been queued
        barrier.await();
        verify(containerBus, timeout(5000).times(2)).handle(isA(ContainerResourceFailedEvent.class));
    } finally {
        dispatcher.stop();
    }
}

Example 23 with DrainDispatcher

use of org.apache.hadoop.yarn.event.DrainDispatcher in project hadoop by apache.

the class TestResourceLocalizationService method testRecovery.

@Test
// mocked generics
@SuppressWarnings("unchecked")
public void testRecovery() throws Exception {
    final String user1 = "user1";
    final String user2 = "user2";
    final ApplicationId appId1 = ApplicationId.newInstance(1, 1);
    final ApplicationId appId2 = ApplicationId.newInstance(1, 2);
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
    stateStore.init(conf);
    stateStore.start();
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    //Ignore actual localization
    EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerBus);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    ResourceLocalizationService spyService = createSpyService(dispatcher, dirsHandler, stateStore);
    try {
        spyService.init(conf);
        spyService.start();
        final Application app1 = mock(Application.class);
        when(app1.getUser()).thenReturn(user1);
        when(app1.getAppId()).thenReturn(appId1);
        final Application app2 = mock(Application.class);
        when(app2.getUser()).thenReturn(user2);
        when(app2.getAppId()).thenReturn(appId2);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app1));
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app2));
        dispatcher.await();
        //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES
        LocalResourcesTracker appTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1);
        LocalResourcesTracker privTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null);
        LocalResourcesTracker appTracker2 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2);
        LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null);
        // init containers
        final Container c1 = getMockContainer(appId1, 1, user1);
        final Container c2 = getMockContainer(appId2, 2, user2);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // Send localization requests of each type.
        final LocalResource privResource1 = getPrivateMockedResource(r);
        final LocalResourceRequest privReq1 = new LocalResourceRequest(privResource1);
        final LocalResource privResource2 = getPrivateMockedResource(r);
        final LocalResourceRequest privReq2 = new LocalResourceRequest(privResource2);
        final LocalResource pubResource1 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq1 = new LocalResourceRequest(pubResource1);
        final LocalResource pubResource2 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2);
        final LocalResource appResource1 = getAppMockedResource(r);
        final LocalResourceRequest appReq1 = new LocalResourceRequest(appResource1);
        final LocalResource appResource2 = getAppMockedResource(r);
        final LocalResourceRequest appReq2 = new LocalResourceRequest(appResource2);
        final LocalResource appResource3 = getAppMockedResource(r);
        final LocalResourceRequest appReq3 = new LocalResourceRequest(appResource3);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req1.put(LocalResourceVisibility.PRIVATE, Arrays.asList(new LocalResourceRequest[] { privReq1, privReq2 }));
        req1.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq1));
        req1.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq1));
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req2.put(LocalResourceVisibility.APPLICATION, Arrays.asList(new LocalResourceRequest[] { appReq2, appReq3 }));
        req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2));
        // Send Request event
        spyService.handle(new ContainerLocalizationRequestEvent(c1, req1));
        spyService.handle(new ContainerLocalizationRequestEvent(c2, req2));
        dispatcher.await();
        // Simulate start of localization for all resources
        privTracker1.getPathForLocalization(privReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null);
        privTracker1.getPathForLocalization(privReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null);
        LocalizedResource privLr1 = privTracker1.getLocalizedResource(privReq1);
        LocalizedResource privLr2 = privTracker1.getLocalizedResource(privReq2);
        appTracker1.getPathForLocalization(appReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId1), null);
        LocalizedResource appLr1 = appTracker1.getLocalizedResource(appReq1);
        appTracker2.getPathForLocalization(appReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null);
        LocalizedResource appLr2 = appTracker2.getLocalizedResource(appReq2);
        appTracker2.getPathForLocalization(appReq3, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null);
        LocalizedResource appLr3 = appTracker2.getLocalizedResource(appReq3);
        pubTracker.getPathForLocalization(pubReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null);
        LocalizedResource pubLr1 = pubTracker.getLocalizedResource(pubReq1);
        pubTracker.getPathForLocalization(pubReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null);
        LocalizedResource pubLr2 = pubTracker.getLocalizedResource(pubReq2);
        // Simulate completion of localization for most resources with
        // possibly different sizes than in the request
        assertNotNull("Localization not started", privLr1.getLocalPath());
        privTracker1.handle(new ResourceLocalizedEvent(privReq1, privLr1.getLocalPath(), privLr1.getSize() + 5));
        assertNotNull("Localization not started", privLr2.getLocalPath());
        privTracker1.handle(new ResourceLocalizedEvent(privReq2, privLr2.getLocalPath(), privLr2.getSize() + 10));
        assertNotNull("Localization not started", appLr1.getLocalPath());
        appTracker1.handle(new ResourceLocalizedEvent(appReq1, appLr1.getLocalPath(), appLr1.getSize()));
        assertNotNull("Localization not started", appLr3.getLocalPath());
        appTracker2.handle(new ResourceLocalizedEvent(appReq3, appLr3.getLocalPath(), appLr3.getSize() + 7));
        assertNotNull("Localization not started", pubLr1.getLocalPath());
        pubTracker.handle(new ResourceLocalizedEvent(pubReq1, pubLr1.getLocalPath(), pubLr1.getSize() + 1000));
        assertNotNull("Localization not started", pubLr2.getLocalPath());
        pubTracker.handle(new ResourceLocalizedEvent(pubReq2, pubLr2.getLocalPath(), pubLr2.getSize() + 99999));
        dispatcher.await();
        assertEquals(ResourceState.LOCALIZED, privLr1.getState());
        assertEquals(ResourceState.LOCALIZED, privLr2.getState());
        assertEquals(ResourceState.LOCALIZED, appLr1.getState());
        assertEquals(ResourceState.DOWNLOADING, appLr2.getState());
        assertEquals(ResourceState.LOCALIZED, appLr3.getState());
        assertEquals(ResourceState.LOCALIZED, pubLr1.getState());
        assertEquals(ResourceState.LOCALIZED, pubLr2.getState());
        // restart and recover
        spyService = createSpyService(dispatcher, dirsHandler, stateStore);
        spyService.init(conf);
        spyService.recoverLocalizedResources(stateStore.loadLocalizationState());
        dispatcher.await();
        appTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1);
        privTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null);
        appTracker2 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2);
        pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null);
        LocalizedResource recoveredRsrc = privTracker1.getLocalizedResource(privReq1);
        assertEquals(privReq1, recoveredRsrc.getRequest());
        assertEquals(privLr1.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(privLr1.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = privTracker1.getLocalizedResource(privReq2);
        assertEquals(privReq2, recoveredRsrc.getRequest());
        assertEquals(privLr2.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(privLr2.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = appTracker1.getLocalizedResource(appReq1);
        assertEquals(appReq1, recoveredRsrc.getRequest());
        assertEquals(appLr1.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(appLr1.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = appTracker2.getLocalizedResource(appReq2);
        assertNull("in-progress resource should not be present", recoveredRsrc);
        recoveredRsrc = appTracker2.getLocalizedResource(appReq3);
        assertEquals(appReq3, recoveredRsrc.getRequest());
        assertEquals(appLr3.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(appLr3.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
    } finally {
        dispatcher.stop();
        stateStore.close();
    }
}

Example 24 with DrainDispatcher

use of org.apache.hadoop.yarn.event.DrainDispatcher in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testLocalResourceCache.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testLocalResourceCache() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<ContainerEvent> containerEventHandler = mock(EventHandler.class);
        // Registering event handlers.
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), null);
        LocalResourceRequest lr = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        // Creating 2 containers for same application which will be requesting
        // same local resource.
        // Container 1 requesting local resource.
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc1);
        // No resource request is initially present in local cache
        Assert.assertEquals(0, localrsrc.size());
        // Container-1 requesting local resource.
        tracker.handle(reqEvent1);
        dispatcher.await();
        // New localized Resource should have been added to local resource map
        // and the requesting container will be added to its waiting queue.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId1));
        Assert.assertEquals(ResourceState.DOWNLOADING, localrsrc.get(lr).getState());
        // Container 2 requesting the resource
        ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
        LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
        ResourceEvent reqEvent2 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc2);
        tracker.handle(reqEvent2);
        dispatcher.await();
        // Container 2 should have been added to the waiting queue of the local
        // resource
        Assert.assertEquals(2, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId2));
        // Failing resource localization
        ResourceEvent resourceFailedEvent = new ResourceFailedLocalizationEvent(lr, (new Exception("test").getMessage()));
        // Backing up the resource to track its state change as it will be
        // removed after the failed event.
        LocalizedResource localizedResource = localrsrc.get(lr);
        tracker.handle(resourceFailedEvent);
        dispatcher.await();
        // After receiving failed resource event; all waiting containers will be
        // notified with Container Resource Failed Event.
        Assert.assertEquals(0, localrsrc.size());
        verify(containerEventHandler, timeout(1000).times(2)).handle(isA(ContainerResourceFailedEvent.class));
        Assert.assertEquals(ResourceState.FAILED, localizedResource.getState());
        // Container 1 trying to release the resource (This resource is already
        // deleted from the cache. This call should return silently without
        // exception.
        ResourceReleaseEvent relEvent1 = new ResourceReleaseEvent(lr, cId1);
        tracker.handle(relEvent1);
        dispatcher.await();
        // Container-3 now requests for the same resource. This request call
        // is coming prior to Container-2's release call.
        ContainerId cId3 = BuilderUtils.newContainerId(1, 1, 1, 3);
        LocalizerContext lc3 = new LocalizerContext(user, cId3, null);
        ResourceEvent reqEvent3 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc3);
        tracker.handle(reqEvent3);
        dispatcher.await();
        // Local resource cache now should have the requested resource and the
        // number of waiting containers should be 1.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
        // Container-2 Releases the resource
        ResourceReleaseEvent relEvent2 = new ResourceReleaseEvent(lr, cId2);
        tracker.handle(relEvent2);
        dispatcher.await();
        // Making sure that there is no change in the cache after the release.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
        // Sending ResourceLocalizedEvent to tracker. In turn resource should
        // send Container Resource Localized Event to waiting containers.
        Path localizedPath = new Path("/tmp/file1");
        ResourceLocalizedEvent localizedEvent = new ResourceLocalizedEvent(lr, localizedPath, 123L);
        tracker.handle(localizedEvent);
        dispatcher.await();
        // Verifying ContainerResourceLocalizedEvent .
        verify(containerEventHandler, timeout(1000).times(1)).handle(isA(ContainerResourceLocalizedEvent.class));
        Assert.assertEquals(ResourceState.LOCALIZED, localrsrc.get(lr).getState());
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        // Container-3 releasing the resource.
        ResourceReleaseEvent relEvent3 = new ResourceReleaseEvent(lr, cId3);
        tracker.handle(relEvent3);
        dispatcher.await();
        Assert.assertEquals(0, localrsrc.get(lr).getRefCount());
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 25 with DrainDispatcher

use of org.apache.hadoop.yarn.event.DrainDispatcher in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testStateStoreSuccessfulLocalization.

@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDir = new Path("/tmp");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    DeletionService mockDelService = mock(DeletionService.class);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore);
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        // Container 1 requests lr1 to be localized
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1);
        tracker.handle(reqEvent1);
        dispatcher.await();
        // Simulate the process of localization of lr1
        Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null);
        ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class);
        ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
        verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture());
        LocalResourceProto lrProto = localResourceCaptor.getValue();
        Path localizedPath1 = pathCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
        Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
        // Simulate lr1 getting localized
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
        tracker.handle(rle1);
        dispatcher.await();
        ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor = ArgumentCaptor.forClass(LocalizedResourceProto.class);
        verify(stateStore).finishResourceLocalization(eq(user), eq(appId), localizedProtoCaptor.capture());
        LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(localizedProto.getResource())));
        Assert.assertEquals(localizedPath1.toString(), localizedProto.getLocalPath());
        LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
        Assert.assertNotNull(localizedRsrc1);
        // simulate release and retention processing
        tracker.handle(new ResourceReleaseEvent(lr1, cId1));
        dispatcher.await();
        boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);
        Assert.assertTrue(removeResult);
        verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}