Examples with LocalizerEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent used on opensource projects

Example 6 with LocalizerEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testResourcePresentInGoodDir.

@SuppressWarnings("unchecked")
@Test
public void testResourcePresentInGoodDir() throws IOException {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        localrsrc.put(req2, lr2);
        LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
        List<String> goodDirs = new ArrayList<String>();
        // /tmp/somedir2 is bad
        goodDirs.add("/tmp/somedir1/");
        goodDirs.add("/tmp/somedir2");
        Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs);
        Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs);
        LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), dirsHandler);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        // Localize R2 for C1
        tracker.handle(req21Event);
        dispatcher.await();
        // Localize resource1
        Path p1 = tracker.getPathForLocalization(req1, new Path("/tmp/somedir1"), null);
        Path p2 = tracker.getPathForLocalization(req2, new Path("/tmp/somedir2"), null);
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1);
        tracker.handle(rle1);
        ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1);
        tracker.handle(rle2);
        dispatcher.await();
        // Remove somedir2 from gooddirs
        Assert.assertTrue(tracker.checkLocalResource(lr2));
        goodDirs.remove(1);
        Assert.assertFalse(tracker.checkLocalResource(lr2));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 7 with LocalizerEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent in project hadoop by apache.

the class TestLocalizedResource method testNotification.

@Test
// mocked generic
@SuppressWarnings("unchecked")
public void testNotification() throws Exception {
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(new Configuration());
    try {
        dispatcher.start();
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
        dispatcher.register(ContainerEventType.class, containerBus);
        dispatcher.register(LocalizerEventType.class, localizerBus);
        // mock resource
        LocalResource apiRsrc = createMockResource();
        final ContainerId container0 = getMockContainer(0L);
        final Credentials creds0 = new Credentials();
        final LocalResourceVisibility vis0 = LocalResourceVisibility.PRIVATE;
        final LocalizerContext ctxt0 = new LocalizerContext("yak", container0, creds0);
        LocalResourceRequest rsrcA = new LocalResourceRequest(apiRsrc);
        LocalizedResource local = new LocalizedResource(rsrcA, dispatcher);
        local.handle(new ResourceRequestEvent(rsrcA, vis0, ctxt0));
        dispatcher.await();
        // Register C0, verify request event
        LocalizerEventMatcher matchesL0Req = new LocalizerEventMatcher(container0, creds0, vis0, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL0Req));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Register C1, verify request event
        final Credentials creds1 = new Credentials();
        final ContainerId container1 = getMockContainer(1L);
        final LocalizerContext ctxt1 = new LocalizerContext("yak", container1, creds1);
        final LocalResourceVisibility vis1 = LocalResourceVisibility.PUBLIC;
        local.handle(new ResourceRequestEvent(rsrcA, vis1, ctxt1));
        dispatcher.await();
        LocalizerEventMatcher matchesL1Req = new LocalizerEventMatcher(container1, creds1, vis1, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL1Req));
        // Release C0 container localization, verify no notification
        local.handle(new ResourceReleaseEvent(rsrcA, container0));
        dispatcher.await();
        verify(containerBus, never()).handle(isA(ContainerEvent.class));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Release C1 container localization, verify no notification
        local.handle(new ResourceReleaseEvent(rsrcA, container1));
        dispatcher.await();
        verify(containerBus, never()).handle(isA(ContainerEvent.class));
        assertEquals(ResourceState.DOWNLOADING, local.getState());
        // Register C2, C3
        final ContainerId container2 = getMockContainer(2L);
        final LocalResourceVisibility vis2 = LocalResourceVisibility.PRIVATE;
        final Credentials creds2 = new Credentials();
        final LocalizerContext ctxt2 = new LocalizerContext("yak", container2, creds2);
        final ContainerId container3 = getMockContainer(3L);
        final LocalResourceVisibility vis3 = LocalResourceVisibility.PRIVATE;
        final Credentials creds3 = new Credentials();
        final LocalizerContext ctxt3 = new LocalizerContext("yak", container3, creds3);
        local.handle(new ResourceRequestEvent(rsrcA, vis2, ctxt2));
        local.handle(new ResourceRequestEvent(rsrcA, vis3, ctxt3));
        dispatcher.await();
        LocalizerEventMatcher matchesL2Req = new LocalizerEventMatcher(container2, creds2, vis2, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL2Req));
        LocalizerEventMatcher matchesL3Req = new LocalizerEventMatcher(container3, creds3, vis3, LocalizerEventType.REQUEST_RESOURCE_LOCALIZATION);
        verify(localizerBus).handle(argThat(matchesL3Req));
        // Successful localization. verify notification C2, C3
        Path locA = new Path("file:///cache/rsrcA");
        local.handle(new ResourceLocalizedEvent(rsrcA, locA, 10));
        dispatcher.await();
        ContainerEventMatcher matchesC2Localized = new ContainerEventMatcher(container2, ContainerEventType.RESOURCE_LOCALIZED);
        ContainerEventMatcher matchesC3Localized = new ContainerEventMatcher(container3, ContainerEventType.RESOURCE_LOCALIZED);
        verify(containerBus).handle(argThat(matchesC2Localized));
        verify(containerBus).handle(argThat(matchesC3Localized));
        assertEquals(ResourceState.LOCALIZED, local.getState());
        // Register C4, verify notification
        final ContainerId container4 = getMockContainer(4L);
        final Credentials creds4 = new Credentials();
        final LocalizerContext ctxt4 = new LocalizerContext("yak", container4, creds4);
        final LocalResourceVisibility vis4 = LocalResourceVisibility.PRIVATE;
        local.handle(new ResourceRequestEvent(rsrcA, vis4, ctxt4));
        dispatcher.await();
        ContainerEventMatcher matchesC4Localized = new ContainerEventMatcher(container4, ContainerEventType.RESOURCE_LOCALIZED);
        verify(containerBus).handle(argThat(matchesC4Localized));
        assertEquals(ResourceState.LOCALIZED, local.getState());
    } finally {
        dispatcher.stop();
    }
}

Example 8 with LocalizerEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent in project hadoop by apache.

the class TestResourceLocalizationService method testResourceRelease.

@Test
// mocked generics
@SuppressWarnings("unchecked")
public void testResourceRelease() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    LocalizerTracker mockLocallilzerTracker = mock(LocalizerTracker.class);
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    //Ignore actual localization
    EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    DeletionService delService = new DeletionService(exec);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker(isA(Configuration.class));
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    try {
        spyService.init(conf);
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES
        LocalResourcesTracker appTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user, appId);
        LocalResourcesTracker privTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId);
        LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // Send localization requests for one resource of each type.
        final LocalResource privResource = getPrivateMockedResource(r);
        final LocalResourceRequest privReq = new LocalResourceRequest(privResource);
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        final LocalResource pubResource2 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2);
        final LocalResource appResource = getAppMockedResource(r);
        final LocalResourceRequest appReq = new LocalResourceRequest(appResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq));
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        req.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq));
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req2.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq));
        req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2));
        Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
        pubRsrcs.add(pubReq);
        pubRsrcs.add(pubReq2);
        // Send Request event
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        spyService.handle(new ContainerLocalizationRequestEvent(c, req2));
        dispatcher.await();
        int privRsrcCount = 0;
        for (LocalizedResource lr : privTracker) {
            privRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 2, lr.getRefCount());
            Assert.assertEquals(privReq, lr.getRequest());
        }
        Assert.assertEquals(1, privRsrcCount);
        int pubRsrcCount = 0;
        for (LocalizedResource lr : pubTracker) {
            pubRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            pubRsrcs.remove(lr.getRequest());
        }
        Assert.assertEquals(0, pubRsrcs.size());
        Assert.assertEquals(2, pubRsrcCount);
        int appRsrcCount = 0;
        for (LocalizedResource lr : appTracker) {
            appRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            Assert.assertEquals(appReq, lr.getRequest());
        }
        Assert.assertEquals(1, appRsrcCount);
        //Send Cleanup Event
        spyService.handle(new ContainerLocalizationCleanupEvent(c, req));
        verify(mockLocallilzerTracker).cleanupPrivLocalizers("container_314159265358979_0003_01_000042");
        req2.remove(LocalResourceVisibility.PRIVATE);
        spyService.handle(new ContainerLocalizationCleanupEvent(c, req2));
        dispatcher.await();
        pubRsrcs.add(pubReq);
        pubRsrcs.add(pubReq2);
        privRsrcCount = 0;
        for (LocalizedResource lr : privTracker) {
            privRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            Assert.assertEquals(privReq, lr.getRequest());
        }
        Assert.assertEquals(1, privRsrcCount);
        pubRsrcCount = 0;
        for (LocalizedResource lr : pubTracker) {
            pubRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 0, lr.getRefCount());
            pubRsrcs.remove(lr.getRequest());
        }
        Assert.assertEquals(0, pubRsrcs.size());
        Assert.assertEquals(2, pubRsrcCount);
        appRsrcCount = 0;
        for (LocalizedResource lr : appTracker) {
            appRsrcCount++;
        }
        Assert.assertEquals(0, appRsrcCount);
    } finally {
        dispatcher.stop();
        delService.stop();
    }
}

Example 9 with LocalizerEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method test.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void test() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        DeletionService mockDelService = mock(DeletionService.class);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
        LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        localrsrc.put(req2, lr2);
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService(), null);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent req12Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2);
        ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);
        ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2);
        ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        // Localize R1 for C2
        tracker.handle(req12Event);
        // Localize R2 for C1
        tracker.handle(req21Event);
        dispatcher.await();
        verify(localizerEventHandler, times(3)).handle(any(LocalizerResourceRequestEvent.class));
        // Verify refCount for R1 is 2
        Assert.assertEquals(2, lr1.getRefCount());
        // Verify refCount for R2 is 1
        Assert.assertEquals(1, lr2.getRefCount());
        // Release R2 for C1
        tracker.handle(rel21Event);
        dispatcher.await();
        verifyTrackedResourceCount(tracker, 2);
        // Verify resource with non zero ref count is not removed.
        Assert.assertEquals(2, lr1.getRefCount());
        Assert.assertFalse(tracker.remove(lr1, mockDelService));
        verifyTrackedResourceCount(tracker, 2);
        // Localize resource1
        ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1);
        lr1.handle(rle);
        Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
        // Release resource1
        tracker.handle(rel11Event);
        tracker.handle(rel12Event);
        Assert.assertEquals(0, lr1.getRefCount());
        // Verify resources in state LOCALIZED with ref-count=0 is removed.
        Assert.assertTrue(tracker.remove(lr1, mockDelService));
        verifyTrackedResourceCount(tracker, 1);
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 10 with LocalizerEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerEvent in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testConsistency.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testConsistency() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService(), null);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        dispatcher.await();
        // Verify refCount for R1 is 1
        Assert.assertEquals(1, lr1.getRefCount());
        dispatcher.await();
        verifyTrackedResourceCount(tracker, 1);
        // Localize resource1
        ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1);
        lr1.handle(rle);
        Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
        Assert.assertTrue(createdummylocalizefile(new Path("file:///tmp/r1")));
        LocalizedResource rsrcbefore = tracker.iterator().next();
        File resFile = new File(lr1.getLocalPath().toUri().getRawPath().toString());
        Assert.assertTrue(resFile.exists());
        Assert.assertTrue(resFile.delete());
        // Localize R1 for C1
        tracker.handle(req11Event);
        dispatcher.await();
        lr1.handle(rle);
        Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
        LocalizedResource rsrcafter = tracker.iterator().next();
        if (rsrcbefore == rsrcafter) {
            Assert.fail("Localized resource should not be equal");
        }
        // Release resource1
        tracker.handle(rel11Event);
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}