Examples with LocalizerResourceRequestEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent used on opensource projects

Example 1 with LocalizerResourceRequestEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizerResourceRequestEvent in project hadoop by apache.

the class TestResourceLocalizationService method testParallelDownloadAttemptsForPublicResource.

@Test(timeout = 100000)
@SuppressWarnings("unchecked")
public void testParallelDownloadAttemptsForPublicResource() throws Exception {
    DrainDispatcher dispatcher1 = null;
    String user = "testuser";
    try {
        // creating one local directory
        List<Path> localDirs = new ArrayList<Path>();
        String[] sDirs = new String[1];
        for (int i = 0; i < 1; ++i) {
            localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
            sDirs[i] = localDirs.get(i).toString();
        }
        conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
        // Registering event handlers
        EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
        dispatcher1 = new DrainDispatcher();
        dispatcher1.register(ApplicationEventType.class, applicationBus);
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        dispatcher1.register(ContainerEventType.class, containerBus);
        ContainerExecutor exec = mock(ContainerExecutor.class);
        DeletionService delService = mock(DeletionService.class);
        LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
        // initializing directory handler.
        dirsHandler.init(conf);
        dispatcher1.init(conf);
        dispatcher1.start();
        // Creating and initializing ResourceLocalizationService but not starting
        // it as otherwise it will remove requests from pending queue.
        ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher1, exec, delService, dirsHandler, nmContext);
        ResourceLocalizationService spyService = spy(rawService);
        dispatcher1.register(LocalizationEventType.class, spyService);
        spyService.init(conf);
        // Initially pending map should be empty for public localizer
        Assert.assertEquals(0, spyService.getPublicLocalizer().pending.size());
        LocalResourceRequest req = new LocalResourceRequest(new Path("/tmp"), 123L, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, "");
        // Initializing application
        ApplicationImpl app = mock(ApplicationImpl.class);
        ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
        when(app.getAppId()).thenReturn(appId);
        when(app.getUser()).thenReturn(user);
        dispatcher1.getEventHandler().handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        // Container - 1
        // container requesting the resource
        ContainerImpl container1 = createMockContainer(user, 1);
        dispatcher1.getEventHandler().handle(createContainerLocalizationEvent(container1, LocalResourceVisibility.PUBLIC, req));
        // Waiting for resource to change into DOWNLOADING state.
        Assert.assertTrue(waitForResourceState(null, spyService, req, LocalResourceVisibility.PUBLIC, user, null, ResourceState.DOWNLOADING, 5000));
        // Waiting for download to start.
        Assert.assertTrue(waitForPublicDownloadToStart(spyService, 1, 5000));
        LocalizedResource lr = getLocalizedResource(spyService, req, LocalResourceVisibility.PUBLIC, user, null);
        // Resource would now have moved into DOWNLOADING state
        Assert.assertEquals(ResourceState.DOWNLOADING, lr.getState());
        // pending should have this resource now.
        Assert.assertEquals(1, spyService.getPublicLocalizer().pending.size());
        // Now resource should have 0 permit.
        Assert.assertEquals(0, lr.sem.availablePermits());
        // Container - 2
        // Container requesting the same resource.
        ContainerImpl container2 = createMockContainer(user, 2);
        dispatcher1.getEventHandler().handle(createContainerLocalizationEvent(container2, LocalResourceVisibility.PUBLIC, req));
        // Waiting for download to start. This should return false as new download
        // will not start
        Assert.assertFalse(waitForPublicDownloadToStart(spyService, 2, 5000));
        // Now Failing the resource download. As a part of it
        // resource state is changed and then lock is released.
        ResourceFailedLocalizationEvent locFailedEvent = new ResourceFailedLocalizationEvent(req, new Exception("test").toString());
        spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, null).handle(locFailedEvent);
        // Waiting for resource to change into FAILED state.
        Assert.assertTrue(waitForResourceState(lr, spyService, req, LocalResourceVisibility.PUBLIC, user, null, ResourceState.FAILED, 5000));
        // releasing lock as a part of download failed process.
        lr.unlock();
        // removing pending download request.
        spyService.getPublicLocalizer().pending.clear();
        LocalizerContext lc = mock(LocalizerContext.class);
        when(lc.getContainerId()).thenReturn(ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(1L, 1), 1), 1L));
        // Now I need to simulate a race condition wherein Event is added to
        // dispatcher before resource state changes to either FAILED or LOCALIZED
        // Hence sending event directly to dispatcher.
        LocalizerResourceRequestEvent localizerEvent = new LocalizerResourceRequestEvent(lr, null, lc, null);
        dispatcher1.getEventHandler().handle(localizerEvent);
        // Waiting for download to start. This should return false as new download
        // will not start
        Assert.assertFalse(waitForPublicDownloadToStart(spyService, 1, 5000));
        // Checking available permits now.
        Assert.assertEquals(1, lr.sem.availablePermits());
    } finally {
        if (dispatcher1 != null) {
            dispatcher1.stop();
        }
    }
}