Examples with LocalizerRunner org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService.LocalizerRunner used on opensource projects

Example 1 with LocalizerRunner

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService.LocalizerRunner in project hadoop by apache.

the class TestResourceLocalizationService method testDownloadingResourcesOnContainerKill.

@Test(timeout = 20000)
@SuppressWarnings("unchecked")
public void testDownloadingResourcesOnContainerKill() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[1];
    localDirs.add(lfs.makeQualified(new Path(basedir, 0 + "")));
    sDirs[0] = localDirs.get(0).toString();
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    DummyExecutor exec = new DummyExecutor();
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    DeletionService delServiceReal = new DeletionService(exec);
    DeletionService delService = spy(delServiceReal);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask());
    FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask());
    final Path userDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.USERCACHE);
    final Path fileDir = new Path(sDirs[0].substring("file:".length()), ContainerLocalizer.FILECACHE);
    final Path sysDir = new Path(sDirs[0].substring("file:".length()), ResourceLocalizationService.NM_PRIVATE_DIR);
    final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", new Path(sDirs[0]));
    final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", sysDir);
    doAnswer(new Answer<FileStatus>() {

        @Override
        public FileStatus answer(InvocationOnMock invocation) throws Throwable {
            Object[] args = invocation.getArguments();
            if (args.length > 0) {
                if (args[0].equals(userDir) || args[0].equals(fileDir)) {
                    return fs;
                }
            }
            return nmFs;
        }
    }).when(spylfs).getFileStatus(isA(Path.class));
    try {
        spyService.init(conf);
        spyService.start();
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        String user = "user0";
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        ArgumentMatcher<ApplicationEvent> matchesAppInit = new ArgumentMatcher<ApplicationEvent>() {

            @Override
            public boolean matches(Object o) {
                ApplicationEvent evt = (ApplicationEvent) o;
                return evt.getType() == ApplicationEventType.APPLICATION_INITED && appId == evt.getApplicationID();
            }
        };
        dispatcher.await();
        verify(applicationBus).handle(argThat(matchesAppInit));
        // Initialize localizer.
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c1 = getMockContainer(appId, 42, "user0");
        final Container c2 = getMockContainer(appId, 43, "user0");
        FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null);
        doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean());
        final LocalResource resource1 = getPrivateMockedResource(r);
        LocalResource resource2 = null;
        do {
            resource2 = getPrivateMockedResource(r);
        } while (resource2 == null || resource2.equals(resource1));
        LocalResource resource3 = null;
        do {
            resource3 = getPrivateMockedResource(r);
        } while (resource3 == null || resource3.equals(resource1) || resource3.equals(resource2));
        // Send localization requests for container c1 and c2.
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
        final LocalResourceRequest req3 = new LocalResourceRequest(resource3);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>();
        privateResourceList.add(req1);
        privateResourceList.add(req2);
        privateResourceList.add(req3);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
        spyService.handle(new ContainerLocalizationRequestEvent(c1, rsrcs));
        final LocalResourceRequest req1_1 = new LocalResourceRequest(resource2);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> privateResourceList1 = new ArrayList<LocalResourceRequest>();
        privateResourceList1.add(req1_1);
        rsrcs1.put(LocalResourceVisibility.PRIVATE, privateResourceList1);
        spyService.handle(new ContainerLocalizationRequestEvent(c2, rsrcs1));
        dispatcher.await();
        // Wait for localizers of both container c1 and c2 to begin.
        exec.waitForLocalizers(2);
        LocalizerRunner locC1 = spyService.getLocalizerRunner(c1.getContainerId().toString());
        final String containerIdStr = c1.getContainerId().toString();
        // Heartbeats from container localizer
        LocalResourceStatus rsrc1success = mock(LocalResourceStatus.class);
        LocalResourceStatus rsrc2pending = mock(LocalResourceStatus.class);
        LocalizerStatus stat = mock(LocalizerStatus.class);
        when(stat.getLocalizerId()).thenReturn(containerIdStr);
        when(rsrc1success.getResource()).thenReturn(resource1);
        when(rsrc2pending.getResource()).thenReturn(resource2);
        when(rsrc1success.getLocalSize()).thenReturn(4344L);
        URL locPath = getPath("/some/path");
        when(rsrc1success.getLocalPath()).thenReturn(locPath);
        when(rsrc1success.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        when(rsrc2pending.getStatus()).thenReturn(ResourceStatusType.FETCH_PENDING);
        when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList()).thenReturn(Collections.singletonList(rsrc1success)).thenReturn(Collections.singletonList(rsrc2pending)).thenReturn(Collections.singletonList(rsrc2pending)).thenReturn(Collections.<LocalResourceStatus>emptyList());
        // First heartbeat which schedules first resource.
        LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        // Second heartbeat which reports first resource as success.
        // Second resource is scheduled.
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        final String locPath1 = response.getResourceSpecs().get(0).getDestinationDirectory().getFile();
        // Third heartbeat which reports second resource as pending.
        // Third resource is scheduled.
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.LIVE, response.getLocalizerAction());
        final String locPath2 = response.getResourceSpecs().get(0).getDestinationDirectory().getFile();
        // Container c1 is killed which leads to cleanup
        spyService.handle(new ContainerLocalizationCleanupEvent(c1, rsrcs));
        // This heartbeat will indicate to container localizer to die as localizer
        // runner has stopped.
        response = spyService.heartbeat(stat);
        assertEquals(LocalizerAction.DIE, response.getLocalizerAction());
        exec.setStopLocalization();
        dispatcher.await();
        // verify container notification
        ArgumentMatcher<ContainerEvent> successContainerLoc = new ArgumentMatcher<ContainerEvent>() {

            @Override
            public boolean matches(Object o) {
                ContainerEvent evt = (ContainerEvent) o;
                return evt.getType() == ContainerEventType.RESOURCE_LOCALIZED && c1.getContainerId() == evt.getContainerID();
            }
        };
        // Only one resource gets localized for container c1.
        verify(containerBus).handle(argThat(successContainerLoc));
        Set<Path> paths = Sets.newHashSet(new Path(locPath1), new Path(locPath1 + "_tmp"), new Path(locPath2), new Path(locPath2 + "_tmp"));
        // Wait for localizer runner thread for container c1 to finish.
        while (locC1.getState() != Thread.State.TERMINATED) {
            Thread.sleep(50);
        }
        // Verify if downloading resources were submitted for deletion.
        verify(delService).delete(eq(user), (Path) eq(null), argThat(new DownloadingPathsMatcher(paths)));
        LocalResourcesTracker tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, "user0", appId);
        // Container c1 was killed but this resource was localized before kill
        // hence its not removed despite ref cnt being 0.
        LocalizedResource rsrc1 = tracker.getLocalizedResource(req1);
        assertNotNull(rsrc1);
        assertEquals(rsrc1.getState(), ResourceState.LOCALIZED);
        assertEquals(rsrc1.getRefCount(), 0);
        // Container c1 was killed but this resource is referenced by container c2
        // as well hence its ref cnt is 1.
        LocalizedResource rsrc2 = tracker.getLocalizedResource(req2);
        assertNotNull(rsrc2);
        assertEquals(rsrc2.getState(), ResourceState.DOWNLOADING);
        assertEquals(rsrc2.getRefCount(), 1);
        // As container c1 was killed and this resource was not referenced by any
        // other container, hence its removed.
        LocalizedResource rsrc3 = tracker.getLocalizedResource(req3);
        assertNull(rsrc3);
    } finally {
        spyService.stop();
        dispatcher.stop();
        delService.stop();
    }
}

Example 2 with LocalizerRunner

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService.LocalizerRunner in project hadoop by apache.

the class TestResourceLocalizationService method testLocalResourcePath.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testLocalResourcePath() throws Exception {
    // test the local path where application and user cache files will be
    // localized.
    DrainDispatcher dispatcher1 = null;
    try {
        dispatcher1 = new DrainDispatcher();
        String user = "testuser";
        ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
        // creating one local directory
        List<Path> localDirs = new ArrayList<Path>();
        String[] sDirs = new String[1];
        for (int i = 0; i < 1; ++i) {
            localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
            sDirs[i] = localDirs.get(i).toString();
        }
        conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
        LocalDirsHandlerService localDirHandler = new LocalDirsHandlerService();
        localDirHandler.init(conf);
        // Registering event handlers
        EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
        dispatcher1.register(ApplicationEventType.class, applicationBus);
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        dispatcher1.register(ContainerEventType.class, containerBus);
        ContainerExecutor exec = mock(ContainerExecutor.class);
        DeletionService delService = mock(DeletionService.class);
        LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
        // initializing directory handler.
        dirsHandler.init(conf);
        dispatcher1.init(conf);
        dispatcher1.start();
        ResourceLocalizationService rls = new ResourceLocalizationService(dispatcher1, exec, delService, localDirHandler, nmContext);
        dispatcher1.register(LocalizationEventType.class, rls);
        rls.init(conf);
        rls.handle(createApplicationLocalizationEvent(user, appId));
        // We need to pre-populate the LocalizerRunner as the
        // Resource Localization Service code internally starts them which
        // definitely we don't want.
        // creating new container and populating corresponding localizer runner
        // Container - 1
        Container container1 = createMockContainer(user, 1);
        String localizerId1 = container1.getContainerId().toString();
        rls.getPrivateLocalizers().put(localizerId1, rls.new LocalizerRunner(new LocalizerContext(user, container1.getContainerId(), null), localizerId1));
        // Creating two requests for container
        // 1) Private resource
        // 2) Application resource
        LocalResourceRequest reqPriv = new LocalResourceRequest(new Path("file:///tmp1"), 123L, LocalResourceType.FILE, LocalResourceVisibility.PRIVATE, "");
        List<LocalResourceRequest> privList = new ArrayList<LocalResourceRequest>();
        privList.add(reqPriv);
        LocalResourceRequest reqApp = new LocalResourceRequest(new Path("file:///tmp2"), 123L, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, "");
        List<LocalResourceRequest> appList = new ArrayList<LocalResourceRequest>();
        appList.add(reqApp);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        rsrcs.put(LocalResourceVisibility.APPLICATION, appList);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privList);
        dispatcher1.getEventHandler().handle(new ContainerLocalizationRequestEvent(container1, rsrcs));
        // Now waiting for resource download to start. Here actual will not start
        // Only the resources will be populated into pending list.
        Assert.assertTrue(waitForPrivateDownloadToStart(rls, localizerId1, 2, 5000));
        // Validating user and application cache paths
        String userCachePath = StringUtils.join(Path.SEPARATOR, Arrays.asList(localDirs.get(0).toUri().getRawPath(), ContainerLocalizer.USERCACHE, user, ContainerLocalizer.FILECACHE));
        String userAppCachePath = StringUtils.join(Path.SEPARATOR, Arrays.asList(localDirs.get(0).toUri().getRawPath(), ContainerLocalizer.USERCACHE, user, ContainerLocalizer.APPCACHE, appId.toString(), ContainerLocalizer.FILECACHE));
        // Now the Application and private resources may come in any order
        // for download.
        // For User cahce :
        // returned destinationPath = user cache path + random number
        // For App cache :
        // returned destinationPath = user app cache path + random number
        int returnedResources = 0;
        boolean appRsrc = false, privRsrc = false;
        while (returnedResources < 2) {
            LocalizerHeartbeatResponse response = rls.heartbeat(createLocalizerStatus(localizerId1));
            for (ResourceLocalizationSpec resourceSpec : response.getResourceSpecs()) {
                returnedResources++;
                Path destinationDirectory = new Path(resourceSpec.getDestinationDirectory().getFile());
                if (resourceSpec.getResource().getVisibility() == LocalResourceVisibility.APPLICATION) {
                    appRsrc = true;
                    Assert.assertEquals(userAppCachePath, destinationDirectory.getParent().toUri().toString());
                } else if (resourceSpec.getResource().getVisibility() == LocalResourceVisibility.PRIVATE) {
                    privRsrc = true;
                    Assert.assertEquals(userCachePath, destinationDirectory.getParent().toUri().toString());
                } else {
                    throw new Exception("Unexpected resource received.");
                }
            }
        }
        // We should receive both the resources (Application and Private)
        Assert.assertTrue(appRsrc && privRsrc);
    } finally {
        if (dispatcher1 != null) {
            dispatcher1.stop();
        }
    }
}

Example 3 with LocalizerRunner

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService.LocalizerRunner in project hadoop by apache.

the class TestResourceLocalizationService method testLocalizerHeartbeatWhenAppCleaningUp.

@Test(timeout = 20000)
@SuppressWarnings("unchecked")
public void testLocalizerHeartbeatWhenAppCleaningUp() throws Exception {
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, lfs.makeQualified(new Path(basedir, 0 + "")).toString());
    // Start dispatcher.
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    dispatcher.register(ApplicationEventType.class, mock(EventHandler.class));
    dispatcher.register(ContainerEventType.class, mock(EventHandler.class));
    DummyExecutor exec = new DummyExecutor();
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    // Start resource localization service.
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, mock(DeletionService.class), dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    try {
        spyService.init(conf);
        spyService.start();
        // Init application resources.
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(1234567890L, 3);
        when(app.getUser()).thenReturn("user0");
        when(app.getAppId()).thenReturn(appId);
        when(app.toString()).thenReturn(appId.toString());
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // Initialize localizer.
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c = getMockContainer(appId, 46, "user0");
        FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null);
        doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean());
        final LocalResource resource1 = getAppMockedResource(r);
        final LocalResource resource2 = getAppMockedResource(r);
        // Send localization requests for container.
        // 2 resources generated with APPLICATION visibility.
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> appResourceList = Arrays.asList(req1, req2);
        rsrcs.put(LocalResourceVisibility.APPLICATION, appResourceList);
        spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
        dispatcher.await();
        // Wait for localization to begin.
        exec.waitForLocalizers(1);
        final String containerIdStr = c.getContainerId().toString();
        LocalizerRunner locRunnerForContainer = spyService.getLocalizerRunner(containerIdStr);
        // Heartbeats from container localizer
        LocalResourceStatus rsrcSuccess = mock(LocalResourceStatus.class);
        LocalizerStatus stat = mock(LocalizerStatus.class);
        when(stat.getLocalizerId()).thenReturn(containerIdStr);
        when(rsrcSuccess.getResource()).thenReturn(resource1);
        when(rsrcSuccess.getLocalSize()).thenReturn(4344L);
        when(rsrcSuccess.getLocalPath()).thenReturn(getPath("/some/path"));
        when(rsrcSuccess.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList());
        // First heartbeat which schedules first resource.
        LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
        assertEquals("NM should tell localizer to be LIVE in Heartbeat.", LocalizerAction.LIVE, response.getLocalizerAction());
        // Cleanup application.
        spyService.handle(new ContainerLocalizationCleanupEvent(c, rsrcs));
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.DESTROY_APPLICATION_RESOURCES, app));
        dispatcher.await();
        try {
            // Directly send heartbeat to introduce race as app is being cleaned up.
            locRunnerForContainer.processHeartbeat(Collections.singletonList(rsrcSuccess));
        } catch (Exception e) {
            fail("Exception should not have been thrown on processing heartbeat");
        }
        // Send another heartbeat.
        response = spyService.heartbeat(stat);
        assertEquals("NM should tell localizer to DIE in Heartbeat.", LocalizerAction.DIE, response.getLocalizerAction());
        exec.setStopLocalization();
    } finally {
        spyService.stop();
        dispatcher.stop();
    }
}

Example 4 with LocalizerRunner

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService.LocalizerRunner in project hadoop by apache.

the class TestResourceLocalizationService method testParallelDownloadAttemptsForPrivateResource.

@Test(timeout = 100000)
@SuppressWarnings("unchecked")
public void testParallelDownloadAttemptsForPrivateResource() throws Exception {
    DrainDispatcher dispatcher1 = null;
    try {
        dispatcher1 = new DrainDispatcher();
        String user = "testuser";
        ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
        // creating one local directory
        List<Path> localDirs = new ArrayList<Path>();
        String[] sDirs = new String[1];
        for (int i = 0; i < 1; ++i) {
            localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
            sDirs[i] = localDirs.get(i).toString();
        }
        conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
        LocalDirsHandlerService localDirHandler = new LocalDirsHandlerService();
        localDirHandler.init(conf);
        // Registering event handlers
        EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
        dispatcher1.register(ApplicationEventType.class, applicationBus);
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        dispatcher1.register(ContainerEventType.class, containerBus);
        ContainerExecutor exec = mock(ContainerExecutor.class);
        DeletionService delService = mock(DeletionService.class);
        LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
        // initializing directory handler.
        dirsHandler.init(conf);
        dispatcher1.init(conf);
        dispatcher1.start();
        ResourceLocalizationService rls = new ResourceLocalizationService(dispatcher1, exec, delService, localDirHandler, nmContext);
        dispatcher1.register(LocalizationEventType.class, rls);
        rls.init(conf);
        rls.handle(createApplicationLocalizationEvent(user, appId));
        LocalResourceRequest req = new LocalResourceRequest(new Path("file:///tmp"), 123L, LocalResourceType.FILE, LocalResourceVisibility.PRIVATE, "");
        // We need to pre-populate the LocalizerRunner as the
        // Resource Localization Service code internally starts them which
        // definitely we don't want.
        // creating new containers and populating corresponding localizer runners
        // Container - 1
        Container container1 = createMockContainer(user, 1);
        String localizerId1 = container1.getContainerId().toString();
        rls.getPrivateLocalizers().put(localizerId1, rls.new LocalizerRunner(new LocalizerContext(user, container1.getContainerId(), null), localizerId1));
        LocalizerRunner localizerRunner1 = rls.getLocalizerRunner(localizerId1);
        dispatcher1.getEventHandler().handle(createContainerLocalizationEvent(container1, LocalResourceVisibility.PRIVATE, req));
        Assert.assertTrue(waitForPrivateDownloadToStart(rls, localizerId1, 1, 5000));
        // Container - 2 now makes the request.
        ContainerImpl container2 = createMockContainer(user, 2);
        String localizerId2 = container2.getContainerId().toString();
        rls.getPrivateLocalizers().put(localizerId2, rls.new LocalizerRunner(new LocalizerContext(user, container2.getContainerId(), null), localizerId2));
        LocalizerRunner localizerRunner2 = rls.getLocalizerRunner(localizerId2);
        dispatcher1.getEventHandler().handle(createContainerLocalizationEvent(container2, LocalResourceVisibility.PRIVATE, req));
        Assert.assertTrue(waitForPrivateDownloadToStart(rls, localizerId2, 1, 5000));
        // Retrieving localized resource.
        LocalResourcesTracker tracker = rls.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId);
        LocalizedResource lr = tracker.getLocalizedResource(req);
        // Resource would now have moved into DOWNLOADING state
        Assert.assertEquals(ResourceState.DOWNLOADING, lr.getState());
        // Resource should have one permit
        Assert.assertEquals(1, lr.sem.availablePermits());
        // Resource Localization Service receives first heart beat from
        // ContainerLocalizer for container1
        LocalizerHeartbeatResponse response1 = rls.heartbeat(createLocalizerStatus(localizerId1));
        // Resource must have been added to scheduled map
        Assert.assertEquals(1, localizerRunner1.scheduled.size());
        // Checking resource in the response and also available permits for it.
        Assert.assertEquals(req.getResource(), response1.getResourceSpecs().get(0).getResource().getResource());
        Assert.assertEquals(0, lr.sem.availablePermits());
        // Resource Localization Service now receives first heart beat from
        // ContainerLocalizer for container2
        LocalizerHeartbeatResponse response2 = rls.heartbeat(createLocalizerStatus(localizerId2));
        // Resource must not have been added to scheduled map
        Assert.assertEquals(0, localizerRunner2.scheduled.size());
        // No resource is returned in response
        Assert.assertEquals(0, response2.getResourceSpecs().size());
        // ContainerLocalizer - 1 now sends failed resource heartbeat.
        rls.heartbeat(createLocalizerStatusForFailedResource(localizerId1, req));
        // Resource Localization should fail and state is modified accordingly.
        // Also Local should be release on the LocalizedResource.
        Assert.assertTrue(waitForResourceState(lr, rls, req, LocalResourceVisibility.PRIVATE, user, appId, ResourceState.FAILED, 5000));
        Assert.assertTrue(lr.getState().equals(ResourceState.FAILED));
        Assert.assertEquals(0, localizerRunner1.scheduled.size());
        // Now Container-2 once again sends heart beat to resource localization
        // service
        // Now container-2 again try to download the resource it should still
        // not get the resource as the resource is now not in DOWNLOADING state.
        response2 = rls.heartbeat(createLocalizerStatus(localizerId2));
        // Resource must not have been added to scheduled map.
        // Also as the resource has failed download it will be removed from
        // pending list.
        Assert.assertEquals(0, localizerRunner2.scheduled.size());
        Assert.assertEquals(0, localizerRunner2.pending.size());
        Assert.assertEquals(0, response2.getResourceSpecs().size());
    } finally {
        if (dispatcher1 != null) {
            dispatcher1.stop();
        }
    }
}