Examples with ResourceLocalizedEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent used on opensource projects

Example 1 with ResourceLocalizedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent in project hadoop by apache.

the class TestResourceLocalizationService method testRecovery.

@Test
// mocked generics
@SuppressWarnings("unchecked")
public void testRecovery() throws Exception {
    final String user1 = "user1";
    final String user2 = "user2";
    final ApplicationId appId1 = ApplicationId.newInstance(1, 1);
    final ApplicationId appId2 = ApplicationId.newInstance(1, 2);
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
    stateStore.init(conf);
    stateStore.start();
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    //Ignore actual localization
    EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerBus);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    ResourceLocalizationService spyService = createSpyService(dispatcher, dirsHandler, stateStore);
    try {
        spyService.init(conf);
        spyService.start();
        final Application app1 = mock(Application.class);
        when(app1.getUser()).thenReturn(user1);
        when(app1.getAppId()).thenReturn(appId1);
        final Application app2 = mock(Application.class);
        when(app2.getUser()).thenReturn(user2);
        when(app2.getAppId()).thenReturn(appId2);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app1));
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app2));
        dispatcher.await();
        //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES
        LocalResourcesTracker appTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1);
        LocalResourcesTracker privTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null);
        LocalResourcesTracker appTracker2 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2);
        LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null);
        // init containers
        final Container c1 = getMockContainer(appId1, 1, user1);
        final Container c2 = getMockContainer(appId2, 2, user2);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // Send localization requests of each type.
        final LocalResource privResource1 = getPrivateMockedResource(r);
        final LocalResourceRequest privReq1 = new LocalResourceRequest(privResource1);
        final LocalResource privResource2 = getPrivateMockedResource(r);
        final LocalResourceRequest privReq2 = new LocalResourceRequest(privResource2);
        final LocalResource pubResource1 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq1 = new LocalResourceRequest(pubResource1);
        final LocalResource pubResource2 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2);
        final LocalResource appResource1 = getAppMockedResource(r);
        final LocalResourceRequest appReq1 = new LocalResourceRequest(appResource1);
        final LocalResource appResource2 = getAppMockedResource(r);
        final LocalResourceRequest appReq2 = new LocalResourceRequest(appResource2);
        final LocalResource appResource3 = getAppMockedResource(r);
        final LocalResourceRequest appReq3 = new LocalResourceRequest(appResource3);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req1.put(LocalResourceVisibility.PRIVATE, Arrays.asList(new LocalResourceRequest[] { privReq1, privReq2 }));
        req1.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq1));
        req1.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq1));
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req2.put(LocalResourceVisibility.APPLICATION, Arrays.asList(new LocalResourceRequest[] { appReq2, appReq3 }));
        req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2));
        // Send Request event
        spyService.handle(new ContainerLocalizationRequestEvent(c1, req1));
        spyService.handle(new ContainerLocalizationRequestEvent(c2, req2));
        dispatcher.await();
        // Simulate start of localization for all resources
        privTracker1.getPathForLocalization(privReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null);
        privTracker1.getPathForLocalization(privReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null);
        LocalizedResource privLr1 = privTracker1.getLocalizedResource(privReq1);
        LocalizedResource privLr2 = privTracker1.getLocalizedResource(privReq2);
        appTracker1.getPathForLocalization(appReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId1), null);
        LocalizedResource appLr1 = appTracker1.getLocalizedResource(appReq1);
        appTracker2.getPathForLocalization(appReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null);
        LocalizedResource appLr2 = appTracker2.getLocalizedResource(appReq2);
        appTracker2.getPathForLocalization(appReq3, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null);
        LocalizedResource appLr3 = appTracker2.getLocalizedResource(appReq3);
        pubTracker.getPathForLocalization(pubReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null);
        LocalizedResource pubLr1 = pubTracker.getLocalizedResource(pubReq1);
        pubTracker.getPathForLocalization(pubReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null);
        LocalizedResource pubLr2 = pubTracker.getLocalizedResource(pubReq2);
        // Simulate completion of localization for most resources with
        // possibly different sizes than in the request
        assertNotNull("Localization not started", privLr1.getLocalPath());
        privTracker1.handle(new ResourceLocalizedEvent(privReq1, privLr1.getLocalPath(), privLr1.getSize() + 5));
        assertNotNull("Localization not started", privLr2.getLocalPath());
        privTracker1.handle(new ResourceLocalizedEvent(privReq2, privLr2.getLocalPath(), privLr2.getSize() + 10));
        assertNotNull("Localization not started", appLr1.getLocalPath());
        appTracker1.handle(new ResourceLocalizedEvent(appReq1, appLr1.getLocalPath(), appLr1.getSize()));
        assertNotNull("Localization not started", appLr3.getLocalPath());
        appTracker2.handle(new ResourceLocalizedEvent(appReq3, appLr3.getLocalPath(), appLr3.getSize() + 7));
        assertNotNull("Localization not started", pubLr1.getLocalPath());
        pubTracker.handle(new ResourceLocalizedEvent(pubReq1, pubLr1.getLocalPath(), pubLr1.getSize() + 1000));
        assertNotNull("Localization not started", pubLr2.getLocalPath());
        pubTracker.handle(new ResourceLocalizedEvent(pubReq2, pubLr2.getLocalPath(), pubLr2.getSize() + 99999));
        dispatcher.await();
        assertEquals(ResourceState.LOCALIZED, privLr1.getState());
        assertEquals(ResourceState.LOCALIZED, privLr2.getState());
        assertEquals(ResourceState.LOCALIZED, appLr1.getState());
        assertEquals(ResourceState.DOWNLOADING, appLr2.getState());
        assertEquals(ResourceState.LOCALIZED, appLr3.getState());
        assertEquals(ResourceState.LOCALIZED, pubLr1.getState());
        assertEquals(ResourceState.LOCALIZED, pubLr2.getState());
        // restart and recover
        spyService = createSpyService(dispatcher, dirsHandler, stateStore);
        spyService.init(conf);
        spyService.recoverLocalizedResources(stateStore.loadLocalizationState());
        dispatcher.await();
        appTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1);
        privTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null);
        appTracker2 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2);
        pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null);
        LocalizedResource recoveredRsrc = privTracker1.getLocalizedResource(privReq1);
        assertEquals(privReq1, recoveredRsrc.getRequest());
        assertEquals(privLr1.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(privLr1.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = privTracker1.getLocalizedResource(privReq2);
        assertEquals(privReq2, recoveredRsrc.getRequest());
        assertEquals(privLr2.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(privLr2.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = appTracker1.getLocalizedResource(appReq1);
        assertEquals(appReq1, recoveredRsrc.getRequest());
        assertEquals(appLr1.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(appLr1.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = appTracker2.getLocalizedResource(appReq2);
        assertNull("in-progress resource should not be present", recoveredRsrc);
        recoveredRsrc = appTracker2.getLocalizedResource(appReq3);
        assertEquals(appReq3, recoveredRsrc.getRequest());
        assertEquals(appLr3.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(appLr3.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
    } finally {
        dispatcher.stop();
        stateStore.close();
    }
}

Example 2 with ResourceLocalizedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testLocalResourceCache.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testLocalResourceCache() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<ContainerEvent> containerEventHandler = mock(EventHandler.class);
        // Registering event handlers.
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), null);
        LocalResourceRequest lr = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        // Creating 2 containers for same application which will be requesting
        // same local resource.
        // Container 1 requesting local resource.
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc1);
        // No resource request is initially present in local cache
        Assert.assertEquals(0, localrsrc.size());
        // Container-1 requesting local resource.
        tracker.handle(reqEvent1);
        dispatcher.await();
        // New localized Resource should have been added to local resource map
        // and the requesting container will be added to its waiting queue.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId1));
        Assert.assertEquals(ResourceState.DOWNLOADING, localrsrc.get(lr).getState());
        // Container 2 requesting the resource
        ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
        LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
        ResourceEvent reqEvent2 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc2);
        tracker.handle(reqEvent2);
        dispatcher.await();
        // Container 2 should have been added to the waiting queue of the local
        // resource
        Assert.assertEquals(2, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId2));
        // Failing resource localization
        ResourceEvent resourceFailedEvent = new ResourceFailedLocalizationEvent(lr, (new Exception("test").getMessage()));
        // Backing up the resource to track its state change as it will be
        // removed after the failed event.
        LocalizedResource localizedResource = localrsrc.get(lr);
        tracker.handle(resourceFailedEvent);
        dispatcher.await();
        // After receiving failed resource event; all waiting containers will be
        // notified with Container Resource Failed Event.
        Assert.assertEquals(0, localrsrc.size());
        verify(containerEventHandler, timeout(1000).times(2)).handle(isA(ContainerResourceFailedEvent.class));
        Assert.assertEquals(ResourceState.FAILED, localizedResource.getState());
        // Container 1 trying to release the resource (This resource is already
        // deleted from the cache. This call should return silently without
        // exception.
        ResourceReleaseEvent relEvent1 = new ResourceReleaseEvent(lr, cId1);
        tracker.handle(relEvent1);
        dispatcher.await();
        // Container-3 now requests for the same resource. This request call
        // is coming prior to Container-2's release call.
        ContainerId cId3 = BuilderUtils.newContainerId(1, 1, 1, 3);
        LocalizerContext lc3 = new LocalizerContext(user, cId3, null);
        ResourceEvent reqEvent3 = new ResourceRequestEvent(lr, LocalResourceVisibility.PRIVATE, lc3);
        tracker.handle(reqEvent3);
        dispatcher.await();
        // Local resource cache now should have the requested resource and the
        // number of waiting containers should be 1.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
        // Container-2 Releases the resource
        ResourceReleaseEvent relEvent2 = new ResourceReleaseEvent(lr, cId2);
        tracker.handle(relEvent2);
        dispatcher.await();
        // Making sure that there is no change in the cache after the release.
        Assert.assertEquals(1, localrsrc.size());
        Assert.assertTrue(localrsrc.containsKey(lr));
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        Assert.assertTrue(localrsrc.get(lr).ref.contains(cId3));
        // Sending ResourceLocalizedEvent to tracker. In turn resource should
        // send Container Resource Localized Event to waiting containers.
        Path localizedPath = new Path("/tmp/file1");
        ResourceLocalizedEvent localizedEvent = new ResourceLocalizedEvent(lr, localizedPath, 123L);
        tracker.handle(localizedEvent);
        dispatcher.await();
        // Verifying ContainerResourceLocalizedEvent .
        verify(containerEventHandler, timeout(1000).times(1)).handle(isA(ContainerResourceLocalizedEvent.class));
        Assert.assertEquals(ResourceState.LOCALIZED, localrsrc.get(lr).getState());
        Assert.assertEquals(1, localrsrc.get(lr).getRefCount());
        // Container-3 releasing the resource.
        ResourceReleaseEvent relEvent3 = new ResourceReleaseEvent(lr, cId3);
        tracker.handle(relEvent3);
        dispatcher.await();
        Assert.assertEquals(0, localrsrc.get(lr).getRefCount());
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 3 with ResourceLocalizedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testStateStoreSuccessfulLocalization.

@Test
@SuppressWarnings("unchecked")
public void testStateStoreSuccessfulLocalization() throws Exception {
    final String user = "someuser";
    final ApplicationId appId = ApplicationId.newInstance(1, 1);
    // This is a random path. NO File creation will take place at this place.
    final Path localDir = new Path("/tmp");
    Configuration conf = new YarnConfiguration();
    DrainDispatcher dispatcher = null;
    dispatcher = createDispatcher(conf);
    EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
    EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerEventHandler);
    dispatcher.register(ContainerEventType.class, containerEventHandler);
    DeletionService mockDelService = mock(DeletionService.class);
    NMStateStoreService stateStore = mock(NMStateStoreService.class);
    try {
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, appId, dispatcher, false, conf, stateStore);
        // Container 1 needs lr1 resource
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalResourceRequest lr1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.APPLICATION);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        // Container 1 requests lr1 to be localized
        ResourceEvent reqEvent1 = new ResourceRequestEvent(lr1, LocalResourceVisibility.APPLICATION, lc1);
        tracker.handle(reqEvent1);
        dispatcher.await();
        // Simulate the process of localization of lr1
        Path hierarchicalPath1 = tracker.getPathForLocalization(lr1, localDir, null);
        ArgumentCaptor<LocalResourceProto> localResourceCaptor = ArgumentCaptor.forClass(LocalResourceProto.class);
        ArgumentCaptor<Path> pathCaptor = ArgumentCaptor.forClass(Path.class);
        verify(stateStore).startResourceLocalization(eq(user), eq(appId), localResourceCaptor.capture(), pathCaptor.capture());
        LocalResourceProto lrProto = localResourceCaptor.getValue();
        Path localizedPath1 = pathCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(lrProto)));
        Assert.assertEquals(hierarchicalPath1, localizedPath1.getParent());
        // Simulate lr1 getting localized
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(lr1, pathCaptor.getValue(), 120);
        tracker.handle(rle1);
        dispatcher.await();
        ArgumentCaptor<LocalizedResourceProto> localizedProtoCaptor = ArgumentCaptor.forClass(LocalizedResourceProto.class);
        verify(stateStore).finishResourceLocalization(eq(user), eq(appId), localizedProtoCaptor.capture());
        LocalizedResourceProto localizedProto = localizedProtoCaptor.getValue();
        Assert.assertEquals(lr1, new LocalResourceRequest(new LocalResourcePBImpl(localizedProto.getResource())));
        Assert.assertEquals(localizedPath1.toString(), localizedProto.getLocalPath());
        LocalizedResource localizedRsrc1 = tracker.getLocalizedResource(lr1);
        Assert.assertNotNull(localizedRsrc1);
        // simulate release and retention processing
        tracker.handle(new ResourceReleaseEvent(lr1, cId1));
        dispatcher.await();
        boolean removeResult = tracker.remove(localizedRsrc1, mockDelService);
        Assert.assertTrue(removeResult);
        verify(stateStore).removeLocalizedResource(eq(user), eq(appId), eq(localizedPath1));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 4 with ResourceLocalizedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testResourcePresentInGoodDir.

@SuppressWarnings("unchecked")
@Test
public void testResourcePresentInGoodDir() throws IOException {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        localrsrc.put(req2, lr2);
        LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
        List<String> goodDirs = new ArrayList<String>();
        // /tmp/somedir2 is bad
        goodDirs.add("/tmp/somedir1/");
        goodDirs.add("/tmp/somedir2");
        Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs);
        Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs);
        LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), dirsHandler);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        // Localize R2 for C1
        tracker.handle(req21Event);
        dispatcher.await();
        // Localize resource1
        Path p1 = tracker.getPathForLocalization(req1, new Path("/tmp/somedir1"), null);
        Path p2 = tracker.getPathForLocalization(req2, new Path("/tmp/somedir2"), null);
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1);
        tracker.handle(rle1);
        ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1);
        tracker.handle(rle2);
        dispatcher.await();
        // Remove somedir2 from gooddirs
        Assert.assertTrue(tracker.checkLocalResource(lr2));
        goodDirs.remove(1);
        Assert.assertFalse(tracker.checkLocalResource(lr2));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 5 with ResourceLocalizedEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ResourceLocalizedEvent in project hadoop by apache.

the class TestLocalizedResource method testNotification.

@Test
// mocked generic
@SuppressWarnings("unchecked")
public void testNotification() throws Exception {
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(new Configuration());
    try {
        dispatcher.start();
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
        dispatcher.register(ContainerEventType.class, containerBus);
        dispatcher.register(LocalizerEventType.class, localizerBus);
        // mock resource
        LocalResource apiRsrc = createMockResource();
        final ContainerId container0 = getMockContainer(0L);
        final Credentials creds0 = new Credentials();
        final LocalResourceVisibility vis0 = LocalResourceVisibility.PRIVATE;
        final LocalizerContext ctxt0 = new LocalizerContext("yak", container0, creds0);
        LocalResourceRequest rsrcA = new LocalResourceRequest(apiRsrc);
        LocalizedResource local = new LocalizedResource(rsrcA, dispatcher);
        local.handle(new ResourceRequestEvent(rsrcA, vis0, ctxt0));
        dispatcher.await();
        // Register C0, verify request event
        LocalizerEventMatcher matchesL0Req = new LocalizerEventMatcher(container0, creds0, vis0, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL0Req));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Register C1, verify request event
        final Credentials creds1 = new Credentials();
        final ContainerId container1 = getMockContainer(1L);
        final LocalizerContext ctxt1 = new LocalizerContext("yak", container1, creds1);
        final LocalResourceVisibility vis1 = LocalResourceVisibility.PUBLIC;
        local.handle(new ResourceRequestEvent(rsrcA, vis1, ctxt1));
        dispatcher.await();
        LocalizerEventMatcher matchesL1Req = new LocalizerEventMatcher(container1, creds1, vis1, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL1Req));
        // Release C0 container localization, verify no notification
        local.handle(new ResourceReleaseEvent(rsrcA, container0));
        dispatcher.await();
        verify(containerBus, never()).handle(isA(ContainerEvent.class));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Release C1 container localization, verify no notification
        local.handle(new ResourceReleaseEvent(rsrcA, container1));
        dispatcher.await();
        verify(containerBus, never()).handle(isA(ContainerEvent.class));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Register C2, C3
        final ContainerId container2 = getMockContainer(2L);
        final LocalResourceVisibility vis2 = LocalResourceVisibility.PRIVATE;
        final Credentials creds2 = new Credentials();
        final LocalizerContext ctxt2 = new LocalizerContext("yak", container2, creds2);
        final ContainerId container3 = getMockContainer(3L);
        final LocalResourceVisibility vis3 = LocalResourceVisibility.PRIVATE;
        final Credentials creds3 = new Credentials();
        final LocalizerContext ctxt3 = new LocalizerContext("yak", container3, creds3);
        local.handle(new ResourceRequestEvent(rsrcA, vis2, ctxt2));
        local.handle(new ResourceRequestEvent(rsrcA, vis3, ctxt3));
        dispatcher.await();
        LocalizerEventMatcher matchesL2Req = new LocalizerEventMatcher(container2, creds2, vis2, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL2Req));
        LocalizerEventMatcher matchesL3Req = new LocalizerEventMatcher(container3, creds3, vis3, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL3Req));
        // Successful localization. verify notification C2, C3
        Path locA = new Path("file:///cache/rsrcA");
        local.handle(new ResourceLocalizedEvent(rsrcA, locA, 10));
        dispatcher.await();
        ContainerEventMatcher matchesC2Localized = new ContainerEventMatcher(container2, ContainerEventType.RESOURCE_LOCALIZED);
        ContainerEventMatcher matchesC3Localized = new ContainerEventMatcher(container3, ContainerEventType.RESOURCE_LOCALIZED);
        verify(containerBus).handle(argThat(matchesC2Localized));
        verify(containerBus).handle(argThat(matchesC3Localized));
        assertEquals(ResourceState.LOCALIZED, local.getState());
        // Register C4, verify notification
        final ContainerId container4 = getMockContainer(4L);
        final Credentials creds4 = new Credentials();
        final LocalizerContext ctxt4 = new LocalizerContext("yak", container4, creds4);
        final LocalResourceVisibility vis4 = LocalResourceVisibility.PRIVATE;
        local.handle(new ResourceRequestEvent(rsrcA, vis4, ctxt4));
        dispatcher.await();
        ContainerEventMatcher matchesC4Localized = new ContainerEventMatcher(container4, ContainerEventType.RESOURCE_LOCALIZED);
        verify(containerBus).handle(argThat(matchesC4Localized));
        assertEquals(ResourceState.LOCALIZED, local.getState());
    } finally {
        dispatcher.stop();
    }
}