Examples with DeletionService org.apache.hadoop.yarn.server.nodemanager.DeletionService used on opensource projects

Example 26 with DeletionService

use of org.apache.hadoop.yarn.server.nodemanager.DeletionService in project hadoop by apache.

the class TestResourceLocalizationService method testResourceRelease.

@Test
// mocked generics
@SuppressWarnings("unchecked")
public void testResourceRelease() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    LocalizerTracker mockLocallilzerTracker = mock(LocalizerTracker.class);
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    //Ignore actual localization
    EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    DeletionService delService = new DeletionService(exec);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker(isA(Configuration.class));
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    try {
        spyService.init(conf);
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES
        LocalResourcesTracker appTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user, appId);
        LocalResourcesTracker privTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId);
        LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // Send localization requests for one resource of each type.
        final LocalResource privResource = getPrivateMockedResource(r);
        final LocalResourceRequest privReq = new LocalResourceRequest(privResource);
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        final LocalResource pubResource2 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2);
        final LocalResource appResource = getAppMockedResource(r);
        final LocalResourceRequest appReq = new LocalResourceRequest(appResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq));
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        req.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq));
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req2.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq));
        req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2));
        Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
        pubRsrcs.add(pubReq);
        pubRsrcs.add(pubReq2);
        // Send Request event
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        spyService.handle(new ContainerLocalizationRequestEvent(c, req2));
        dispatcher.await();
        int privRsrcCount = 0;
        for (LocalizedResource lr : privTracker) {
            privRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 2, lr.getRefCount());
            Assert.assertEquals(privReq, lr.getRequest());
        }
        Assert.assertEquals(1, privRsrcCount);
        int pubRsrcCount = 0;
        for (LocalizedResource lr : pubTracker) {
            pubRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            pubRsrcs.remove(lr.getRequest());
        }
        Assert.assertEquals(0, pubRsrcs.size());
        Assert.assertEquals(2, pubRsrcCount);
        int appRsrcCount = 0;
        for (LocalizedResource lr : appTracker) {
            appRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            Assert.assertEquals(appReq, lr.getRequest());
        }
        Assert.assertEquals(1, appRsrcCount);
        //Send Cleanup Event
        spyService.handle(new ContainerLocalizationCleanupEvent(c, req));
        verify(mockLocallilzerTracker).cleanupPrivLocalizers("container_314159265358979_0003_01_000042");
        req2.remove(LocalResourceVisibility.PRIVATE);
        spyService.handle(new ContainerLocalizationCleanupEvent(c, req2));
        dispatcher.await();
        pubRsrcs.add(pubReq);
        pubRsrcs.add(pubReq2);
        privRsrcCount = 0;
        for (LocalizedResource lr : privTracker) {
            privRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            Assert.assertEquals(privReq, lr.getRequest());
        }
        Assert.assertEquals(1, privRsrcCount);
        pubRsrcCount = 0;
        for (LocalizedResource lr : pubTracker) {
            pubRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 0, lr.getRefCount());
            pubRsrcs.remove(lr.getRequest());
        }
        Assert.assertEquals(0, pubRsrcs.size());
        Assert.assertEquals(2, pubRsrcCount);
        appRsrcCount = 0;
        for (LocalizedResource lr : appTracker) {
            appRsrcCount++;
        }
        Assert.assertEquals(0, appRsrcCount);
    } finally {
        dispatcher.stop();
        delService.stop();
    }
}

Example 27 with DeletionService

use of org.apache.hadoop.yarn.server.nodemanager.DeletionService in project hadoop by apache.

the class TestResourceLocalizationService method createSpyService.

private ResourceLocalizationService createSpyService(DrainDispatcher dispatcher, LocalDirsHandlerService dirsHandler, NMStateStoreService stateStore) {
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalizerTracker mockLocalizerTracker = mock(LocalizerTracker.class);
    DeletionService delService = mock(DeletionService.class);
    NMContext nmContext = new NMContext(new NMContainerTokenSecretManager(conf), new NMTokenSecretManagerInNM(), null, new ApplicationACLsManager(conf), stateStore, false, conf);
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(mockLocalizerTracker).when(spyService).createLocalizerTracker(isA(Configuration.class));
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    return spyService;
}

Example 28 with DeletionService

use of org.apache.hadoop.yarn.server.nodemanager.DeletionService in project hadoop by apache.

the class TestResourceLocalizationService method testParallelDownloadAttemptsForPrivateResource.

@Test(timeout = 100000)
@SuppressWarnings("unchecked")
public void testParallelDownloadAttemptsForPrivateResource() throws Exception {
    DrainDispatcher dispatcher1 = null;
    try {
        dispatcher1 = new DrainDispatcher();
        String user = "testuser";
        ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
        // creating one local directory
        List<Path> localDirs = new ArrayList<Path>();
        String[] sDirs = new String[1];
        for (int i = 0; i < 1; ++i) {
            localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
            sDirs[i] = localDirs.get(i).toString();
        }
        conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
        LocalDirsHandlerService localDirHandler = new LocalDirsHandlerService();
        localDirHandler.init(conf);
        // Registering event handlers
        EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
        dispatcher1.register(ApplicationEventType.class, applicationBus);
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        dispatcher1.register(ContainerEventType.class, containerBus);
        ContainerExecutor exec = mock(ContainerExecutor.class);
        DeletionService delService = mock(DeletionService.class);
        LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
        // initializing directory handler.
        dirsHandler.init(conf);
        dispatcher1.init(conf);
        dispatcher1.start();
        ResourceLocalizationService rls = new ResourceLocalizationService(dispatcher1, exec, delService, localDirHandler, nmContext);
        dispatcher1.register(LocalizationEventType.class, rls);
        rls.init(conf);
        rls.handle(createApplicationLocalizationEvent(user, appId));
        LocalResourceRequest req = new LocalResourceRequest(new Path("file:///tmp"), 123L, LocalResourceType.FILE, LocalResourceVisibility.PRIVATE, "");
        // We need to pre-populate the LocalizerRunner as the
        // Resource Localization Service code internally starts them which
        // definitely we don't want.
        // creating new containers and populating corresponding localizer runners
        // Container - 1
        Container container1 = createMockContainer(user, 1);
        String localizerId1 = container1.getContainerId().toString();
        rls.getPrivateLocalizers().put(localizerId1, rls.new LocalizerRunner(new LocalizerContext(user, container1.getContainerId(), null), localizerId1));
        LocalizerRunner localizerRunner1 = rls.getLocalizerRunner(localizerId1);
        dispatcher1.getEventHandler().handle(createContainerLocalizationEvent(container1, LocalResourceVisibility.PRIVATE, req));
        Assert.assertTrue(waitForPrivateDownloadToStart(rls, localizerId1, 1, 5000));
        // Container - 2 now makes the request.
        ContainerImpl container2 = createMockContainer(user, 2);
        String localizerId2 = container2.getContainerId().toString();
        rls.getPrivateLocalizers().put(localizerId2, rls.new LocalizerRunner(new LocalizerContext(user, container2.getContainerId(), null), localizerId2));
        LocalizerRunner localizerRunner2 = rls.getLocalizerRunner(localizerId2);
        dispatcher1.getEventHandler().handle(createContainerLocalizationEvent(container2, LocalResourceVisibility.PRIVATE, req));
        Assert.assertTrue(waitForPrivateDownloadToStart(rls, localizerId2, 1, 5000));
        // Retrieving localized resource.
        LocalResourcesTracker tracker = rls.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId);
        LocalizedResource lr = tracker.getLocalizedResource(req);
        // Resource would now have moved into DOWNLOADING state
        Assert.assertEquals(ResourceState.DOWNLOADING, lr.getState());
        // Resource should have one permit
        Assert.assertEquals(1, lr.sem.availablePermits());
        // Resource Localization Service receives first heart beat from
        // ContainerLocalizer for container1
        LocalizerHeartbeatResponse response1 = rls.heartbeat(createLocalizerStatus(localizerId1));
        // Resource must have been added to scheduled map
        Assert.assertEquals(1, localizerRunner1.scheduled.size());
        // Checking resource in the response and also available permits for it.
        Assert.assertEquals(req.getResource(), response1.getResourceSpecs().get(0).getResource().getResource());
        Assert.assertEquals(0, lr.sem.availablePermits());
        // Resource Localization Service now receives first heart beat from
        // ContainerLocalizer for container2
        LocalizerHeartbeatResponse response2 = rls.heartbeat(createLocalizerStatus(localizerId2));
        // Resource must not have been added to scheduled map
        Assert.assertEquals(0, localizerRunner2.scheduled.size());
        // No resource is returned in response
        Assert.assertEquals(0, response2.getResourceSpecs().size());
        // ContainerLocalizer - 1 now sends failed resource heartbeat.
        rls.heartbeat(createLocalizerStatusForFailedResource(localizerId1, req));
        // Resource Localization should fail and state is modified accordingly.
        // Also Local should be release on the LocalizedResource.
        Assert.assertTrue(waitForResourceState(lr, rls, req, LocalResourceVisibility.PRIVATE, user, appId, ResourceState.FAILED, 5000));
        Assert.assertTrue(lr.getState().equals(ResourceState.FAILED));
        Assert.assertEquals(0, localizerRunner1.scheduled.size());
        // Now Container-2 once again sends heart beat to resource localization
        // service
        // Now container-2 again try to download the resource it should still
        // not get the resource as the resource is now not in DOWNLOADING state.
        response2 = rls.heartbeat(createLocalizerStatus(localizerId2));
        // Resource must not have been added to scheduled map.
        // Also as the resource has failed download it will be removed from
        // pending list.
        Assert.assertEquals(0, localizerRunner2.scheduled.size());
        Assert.assertEquals(0, localizerRunner2.pending.size());
        Assert.assertEquals(0, response2.getResourceSpecs().size());
    } finally {
        if (dispatcher1 != null) {
            dispatcher1.stop();
        }
    }
}

Example 29 with DeletionService

use of org.apache.hadoop.yarn.server.nodemanager.DeletionService in project hadoop by apache.

the class TestResourceLocalizationService method testDirectoryCleanupOnNewlyCreatedStateStore.

@Test
public void testDirectoryCleanupOnNewlyCreatedStateStore() throws IOException, URISyntaxException {
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    AsyncDispatcher dispatcher = new AsyncDispatcher();
    dispatcher.init(new Configuration());
    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = spy(new DeletionService(exec));
    delService.init(conf);
    delService.start();
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    LocalDirsHandlerService diskhandler = new LocalDirsHandlerService();
    diskhandler.init(conf);
    NMStateStoreService nmStateStoreService = mock(NMStateStoreService.class);
    when(nmStateStoreService.canRecover()).thenReturn(true);
    when(nmStateStoreService.isNewlyCreated()).thenReturn(true);
    ResourceLocalizationService locService = spy(new ResourceLocalizationService(dispatcher, exec, delService, diskhandler, nmContext));
    doReturn(lfs).when(locService).getLocalFileContext(isA(Configuration.class));
    try {
        dispatcher.start();
        // initialize ResourceLocalizationService
        locService.init(conf);
        final FsPermission defaultPerm = new FsPermission((short) 0755);
        // verify directory creation
        for (Path p : localDirs) {
            p = new Path((new URI(p.toString())).getPath());
            Path usercache = new Path(p, ContainerLocalizer.USERCACHE);
            verify(spylfs).rename(eq(usercache), any(Path.class), any(Options.Rename.class));
            verify(spylfs).mkdir(eq(usercache), eq(defaultPerm), eq(true));
            Path publicCache = new Path(p, ContainerLocalizer.FILECACHE);
            verify(spylfs).rename(eq(usercache), any(Path.class), any(Options.Rename.class));
            verify(spylfs).mkdir(eq(publicCache), eq(defaultPerm), eq(true));
            Path nmPriv = new Path(p, ResourceLocalizationService.NM_PRIVATE_DIR);
            verify(spylfs).rename(eq(usercache), any(Path.class), any(Options.Rename.class));
            verify(spylfs).mkdir(eq(nmPriv), eq(ResourceLocalizationService.NM_PRIVATE_PERM), eq(true));
        }
    } finally {
        dispatcher.stop();
        delService.stop();
    }
}

Example 30 with DeletionService

use of org.apache.hadoop.yarn.server.nodemanager.DeletionService in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method test.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void test() {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        DeletionService mockDelService = mock(DeletionService.class);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        ContainerId cId2 = BuilderUtils.newContainerId(1, 1, 1, 2);
        LocalizerContext lc2 = new LocalizerContext(user, cId2, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        localrsrc.put(req2, lr2);
        LocalResourcesTracker tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, false, conf, new NMNullStateStoreService(), null);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent req12Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc2);
        ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent rel11Event = new ResourceReleaseEvent(req1, cId1);
        ResourceEvent rel12Event = new ResourceReleaseEvent(req1, cId2);
        ResourceEvent rel21Event = new ResourceReleaseEvent(req2, cId1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        // Localize R1 for C2
        tracker.handle(req12Event);
        // Localize R2 for C1
        tracker.handle(req21Event);
        dispatcher.await();
        verify(localizerEventHandler, times(3)).handle(any(LocalizerResourceRequestEvent.class));
        // Verify refCount for R1 is 2
        Assert.assertEquals(2, lr1.getRefCount());
        // Verify refCount for R2 is 1
        Assert.assertEquals(1, lr2.getRefCount());
        // Release R2 for C1
        tracker.handle(rel21Event);
        dispatcher.await();
        verifyTrackedResourceCount(tracker, 2);
        // Verify resource with non zero ref count is not removed.
        Assert.assertEquals(2, lr1.getRefCount());
        Assert.assertFalse(tracker.remove(lr1, mockDelService));
        verifyTrackedResourceCount(tracker, 2);
        // Localize resource1
        ResourceLocalizedEvent rle = new ResourceLocalizedEvent(req1, new Path("file:///tmp/r1"), 1);
        lr1.handle(rle);
        Assert.assertTrue(lr1.getState().equals(ResourceState.LOCALIZED));
        // Release resource1
        tracker.handle(rel11Event);
        tracker.handle(rel12Event);
        Assert.assertEquals(0, lr1.getRefCount());
        // Verify resources in state LOCALIZED with ref-count=0 is removed.
        Assert.assertTrue(tracker.remove(lr1, mockDelService));
        verifyTrackedResourceCount(tracker, 1);
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}