Examples with ContainerLocalizationRequestEvent org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent used on opensource projects

Example 6 with ContainerLocalizationRequestEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent in project hadoop by apache.

the class TestResourceLocalizationService method testPublicResourceAddResourceExceptions.

/*
   * Test case for handling RejectedExecutionException and IOException which can
   * be thrown when adding public resources to the pending queue.
   * RejectedExecutionException can be thrown either due to the incoming queue
   * being full or if the ExecutorCompletionService threadpool is shutdown.
   * Since it's hard to simulate the queue being full, this test just shuts down
   * the threadpool and makes sure the exception is handled. If anything is
   * messed up the async dispatcher thread will cause a system exit causing the
   * test to fail.
   */
@Test
@SuppressWarnings("unchecked")
public void testPublicResourceAddResourceExceptions() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
    DrainDispatcher dispatcher = new DrainDispatcher();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = mock(DeletionService.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
    dirsHandlerSpy.init(conf);
    dispatcher.init(conf);
    dispatcher.start();
    try {
        ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext);
        ResourceLocalizationService spyService = spy(rawService);
        doReturn(mockServer).when(spyService).createServer();
        doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
        spyService.init(conf);
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // init resources
        Random r = new Random();
        r.setSeed(r.nextLong());
        // Queue localization request for the public resource
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // first test ioexception
        Mockito.doThrow(new IOException()).when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class), Mockito.anyLong(), Mockito.anyBoolean());
        // send request
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        dispatcher.await();
        LocalResourcesTracker tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
        Assert.assertNull(tracker.getLocalizedResource(pubReq));
        // test IllegalArgumentException
        String name = Long.toHexString(r.nextLong());
        URL url = getPath("/local/PRIVATE/" + name + "/");
        final LocalResource rsrc = BuilderUtils.newLocalResource(url, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
        final LocalResourceRequest pubReq1 = new LocalResourceRequest(rsrc);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req1.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq1));
        Mockito.doCallRealMethod().when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class), Mockito.anyLong(), Mockito.anyBoolean());
        // send request
        spyService.handle(new ContainerLocalizationRequestEvent(c, req1));
        dispatcher.await();
        tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
        Assert.assertNull(tracker.getLocalizedResource(pubReq));
        // test RejectedExecutionException by shutting down the thread pool
        PublicLocalizer publicLocalizer = spyService.getPublicLocalizer();
        publicLocalizer.threadPool.shutdown();
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        dispatcher.await();
        tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
        Assert.assertNull(tracker.getLocalizedResource(pubReq));
    } finally {
        // if we call stop with events in the queue, an InterruptedException gets
        // thrown resulting in the dispatcher thread causing a system exit
        dispatcher.await();
        dispatcher.stop();
    }
}

Example 7 with ContainerLocalizationRequestEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent in project hadoop by apache.

the class TestResourceLocalizationService method testLocalResourcePath.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testLocalResourcePath() throws Exception {
    // test the local path where application and user cache files will be
    // localized.
    DrainDispatcher dispatcher1 = null;
    try {
        dispatcher1 = new DrainDispatcher();
        String user = "testuser";
        ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
        // creating one local directory
        List<Path> localDirs = new ArrayList<Path>();
        String[] sDirs = new String[1];
        for (int i = 0; i < 1; ++i) {
            localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
            sDirs[i] = localDirs.get(i).toString();
        }
        conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
        LocalDirsHandlerService localDirHandler = new LocalDirsHandlerService();
        localDirHandler.init(conf);
        // Registering event handlers
        EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
        dispatcher1.register(ApplicationEventType.class, applicationBus);
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        dispatcher1.register(ContainerEventType.class, containerBus);
        ContainerExecutor exec = mock(ContainerExecutor.class);
        DeletionService delService = mock(DeletionService.class);
        LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
        // initializing directory handler.
        dirsHandler.init(conf);
        dispatcher1.init(conf);
        dispatcher1.start();
        ResourceLocalizationService rls = new ResourceLocalizationService(dispatcher1, exec, delService, localDirHandler, nmContext);
        dispatcher1.register(LocalizationEventType.class, rls);
        rls.init(conf);
        rls.handle(createApplicationLocalizationEvent(user, appId));
        // We need to pre-populate the LocalizerRunner as the
        // Resource Localization Service code internally starts them which
        // definitely we don't want.
        // creating new container and populating corresponding localizer runner
        // Container - 1
        Container container1 = createMockContainer(user, 1);
        String localizerId1 = container1.getContainerId().toString();
        rls.getPrivateLocalizers().put(localizerId1, rls.new LocalizerRunner(new LocalizerContext(user, container1.getContainerId(), null), localizerId1));
        // Creating two requests for container
        // 1) Private resource
        // 2) Application resource
        LocalResourceRequest reqPriv = new LocalResourceRequest(new Path("file:///tmp1"), 123L, LocalResourceType.FILE, LocalResourceVisibility.PRIVATE, "");
        List<LocalResourceRequest> privList = new ArrayList<LocalResourceRequest>();
        privList.add(reqPriv);
        LocalResourceRequest reqApp = new LocalResourceRequest(new Path("file:///tmp2"), 123L, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, "");
        List<LocalResourceRequest> appList = new ArrayList<LocalResourceRequest>();
        appList.add(reqApp);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        rsrcs.put(LocalResourceVisibility.APPLICATION, appList);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privList);
        dispatcher1.getEventHandler().handle(new ContainerLocalizationRequestEvent(container1, rsrcs));
        // Now waiting for resource download to start. Here actual will not start
        // Only the resources will be populated into pending list.
        Assert.assertTrue(waitForPrivateDownloadToStart(rls, localizerId1, 2, 5000));
        // Validating user and application cache paths
        String userCachePath = StringUtils.join(Path.SEPARATOR, Arrays.asList(localDirs.get(0).toUri().getRawPath(), ContainerLocalizer.USERCACHE, user, ContainerLocalizer.FILECACHE));
        String userAppCachePath = StringUtils.join(Path.SEPARATOR, Arrays.asList(localDirs.get(0).toUri().getRawPath(), ContainerLocalizer.USERCACHE, user, ContainerLocalizer.APPCACHE, appId.toString(), ContainerLocalizer.FILECACHE));
        // Now the Application and private resources may come in any order
        // for download.
        // For User cahce :
        // returned destinationPath = user cache path + random number
        // For App cache :
        // returned destinationPath = user app cache path + random number
        int returnedResources = 0;
        boolean appRsrc = false, privRsrc = false;
        while (returnedResources < 2) {
            LocalizerHeartbeatResponse response = rls.heartbeat(createLocalizerStatus(localizerId1));
            for (ResourceLocalizationSpec resourceSpec : response.getResourceSpecs()) {
                returnedResources++;
                Path destinationDirectory = new Path(resourceSpec.getDestinationDirectory().getFile());
                if (resourceSpec.getResource().getVisibility() == LocalResourceVisibility.APPLICATION) {
                    appRsrc = true;
                    Assert.assertEquals(userAppCachePath, destinationDirectory.getParent().toUri().toString());
                } else if (resourceSpec.getResource().getVisibility() == LocalResourceVisibility.PRIVATE) {
                    privRsrc = true;
                    Assert.assertEquals(userCachePath, destinationDirectory.getParent().toUri().toString());
                } else {
                    throw new Exception("Unexpected resource received.");
                }
            }
        }
        // We should receive both the resources (Application and Private)
        Assert.assertTrue(appRsrc && privRsrc);
    } finally {
        if (dispatcher1 != null) {
            dispatcher1.stop();
        }
    }
}

Example 8 with ContainerLocalizationRequestEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent in project hadoop by apache.

the class TestResourceLocalizationService method testLocalizerRunnerException.

@Test(timeout = 10000)
// mocked generics
@SuppressWarnings("unchecked")
public void testLocalizerRunnerException() throws Exception {
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
    dirsHandlerSpy.init(conf);
    DeletionService delServiceReal = new DeletionService(exec);
    DeletionService delService = spy(delServiceReal);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    try {
        spyService.init(conf);
        spyService.start();
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn("user0");
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c = getMockContainer(appId, 42, "user0");
        final LocalResource resource1 = getPrivateMockedResource(r);
        System.out.println("Here 4");
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>();
        privateResourceList.add(req1);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
        final Constructor<?>[] constructors = FSError.class.getDeclaredConstructors();
        constructors[0].setAccessible(true);
        FSError fsError = (FSError) constructors[0].newInstance(new IOException("Disk Error"));
        Mockito.doThrow(fsError).when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class));
        spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
        Thread.sleep(1000);
        dispatcher.await();
        // Verify if ContainerResourceFailedEvent is invoked on FSError
        verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
    } finally {
        spyService.stop();
        dispatcher.stop();
        delService.stop();
    }
}

Example 9 with ContainerLocalizationRequestEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent in project hadoop by apache.

the class TestResourceLocalizationService method testFailedPublicResource.

@Test(timeout = 20000)
// mocked generics
@SuppressWarnings("unchecked")
public void testFailedPublicResource() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    DrainDispatcher dispatcher = new DrainDispatcher();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = mock(DeletionService.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    dispatcher.init(conf);
    dispatcher.start();
    try {
        ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandler, nmContext);
        ResourceLocalizationService spyService = spy(rawService);
        doReturn(mockServer).when(spyService).createServer();
        doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
        spyService.init(conf);
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // cause chmod to fail after a delay
        final CyclicBarrier barrier = new CyclicBarrier(2);
        doAnswer(new Answer<Void>() {

            public Void answer(InvocationOnMock invocation) throws IOException {
                try {
                    barrier.await();
                } catch (InterruptedException e) {
                } catch (BrokenBarrierException e) {
                }
                throw new IOException("forced failure");
            }
        }).when(spylfs).setPermission(isA(Path.class), isA(FsPermission.class));
        // Queue up two localization requests for the same public resource
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        Set<LocalResourceRequest> pubRsrcs = new HashSet<LocalResourceRequest>();
        pubRsrcs.add(pubReq);
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        dispatcher.await();
        // allow the chmod to fail now that both requests have been queued
        barrier.await();
        verify(containerBus, timeout(5000).times(2)).handle(isA(ContainerResourceFailedEvent.class));
    } finally {
        dispatcher.stop();
    }
}

Example 10 with ContainerLocalizationRequestEvent

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ContainerLocalizationRequestEvent in project hadoop by apache.

the class TestResourceLocalizationService method testRecovery.

@Test
// mocked generics
@SuppressWarnings("unchecked")
public void testRecovery() throws Exception {
    final String user1 = "user1";
    final String user2 = "user2";
    final ApplicationId appId1 = ApplicationId.newInstance(1, 1);
    final ApplicationId appId2 = ApplicationId.newInstance(1, 2);
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    conf.setBoolean(YarnConfiguration.NM_RECOVERY_ENABLED, true);
    NMMemoryStateStoreService stateStore = new NMMemoryStateStoreService();
    stateStore.init(conf);
    stateStore.start();
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    //Ignore actual localization
    EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerBus);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    ResourceLocalizationService spyService = createSpyService(dispatcher, dirsHandler, stateStore);
    try {
        spyService.init(conf);
        spyService.start();
        final Application app1 = mock(Application.class);
        when(app1.getUser()).thenReturn(user1);
        when(app1.getAppId()).thenReturn(appId1);
        final Application app2 = mock(Application.class);
        when(app2.getUser()).thenReturn(user2);
        when(app2.getAppId()).thenReturn(appId2);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app1));
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app2));
        dispatcher.await();
        //Get a handle on the trackers after they're setup with INIT_APP_RESOURCES
        LocalResourcesTracker appTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1);
        LocalResourcesTracker privTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null);
        LocalResourcesTracker appTracker2 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2);
        LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null);
        // init containers
        final Container c1 = getMockContainer(appId1, 1, user1);
        final Container c2 = getMockContainer(appId2, 2, user2);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        // Send localization requests of each type.
        final LocalResource privResource1 = getPrivateMockedResource(r);
        final LocalResourceRequest privReq1 = new LocalResourceRequest(privResource1);
        final LocalResource privResource2 = getPrivateMockedResource(r);
        final LocalResourceRequest privReq2 = new LocalResourceRequest(privResource2);
        final LocalResource pubResource1 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq1 = new LocalResourceRequest(pubResource1);
        final LocalResource pubResource2 = getPublicMockedResource(r);
        final LocalResourceRequest pubReq2 = new LocalResourceRequest(pubResource2);
        final LocalResource appResource1 = getAppMockedResource(r);
        final LocalResourceRequest appReq1 = new LocalResourceRequest(appResource1);
        final LocalResource appResource2 = getAppMockedResource(r);
        final LocalResourceRequest appReq2 = new LocalResourceRequest(appResource2);
        final LocalResource appResource3 = getAppMockedResource(r);
        final LocalResourceRequest appReq3 = new LocalResourceRequest(appResource3);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req1.put(LocalResourceVisibility.PRIVATE, Arrays.asList(new LocalResourceRequest[] { privReq1, privReq2 }));
        req1.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq1));
        req1.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq1));
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req2.put(LocalResourceVisibility.APPLICATION, Arrays.asList(new LocalResourceRequest[] { appReq2, appReq3 }));
        req2.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq2));
        // Send Request event
        spyService.handle(new ContainerLocalizationRequestEvent(c1, req1));
        spyService.handle(new ContainerLocalizationRequestEvent(c2, req2));
        dispatcher.await();
        // Simulate start of localization for all resources
        privTracker1.getPathForLocalization(privReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null);
        privTracker1.getPathForLocalization(privReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + user1), null);
        LocalizedResource privLr1 = privTracker1.getLocalizedResource(privReq1);
        LocalizedResource privLr2 = privTracker1.getLocalizedResource(privReq2);
        appTracker1.getPathForLocalization(appReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId1), null);
        LocalizedResource appLr1 = appTracker1.getLocalizedResource(appReq1);
        appTracker2.getPathForLocalization(appReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null);
        LocalizedResource appLr2 = appTracker2.getLocalizedResource(appReq2);
        appTracker2.getPathForLocalization(appReq3, dirsHandler.getLocalPathForWrite(ContainerLocalizer.APPCACHE + appId2), null);
        LocalizedResource appLr3 = appTracker2.getLocalizedResource(appReq3);
        pubTracker.getPathForLocalization(pubReq1, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null);
        LocalizedResource pubLr1 = pubTracker.getLocalizedResource(pubReq1);
        pubTracker.getPathForLocalization(pubReq2, dirsHandler.getLocalPathForWrite(ContainerLocalizer.FILECACHE), null);
        LocalizedResource pubLr2 = pubTracker.getLocalizedResource(pubReq2);
        // Simulate completion of localization for most resources with
        // possibly different sizes than in the request
        assertNotNull("Localization not started", privLr1.getLocalPath());
        privTracker1.handle(new ResourceLocalizedEvent(privReq1, privLr1.getLocalPath(), privLr1.getSize() + 5));
        assertNotNull("Localization not started", privLr2.getLocalPath());
        privTracker1.handle(new ResourceLocalizedEvent(privReq2, privLr2.getLocalPath(), privLr2.getSize() + 10));
        assertNotNull("Localization not started", appLr1.getLocalPath());
        appTracker1.handle(new ResourceLocalizedEvent(appReq1, appLr1.getLocalPath(), appLr1.getSize()));
        assertNotNull("Localization not started", appLr3.getLocalPath());
        appTracker2.handle(new ResourceLocalizedEvent(appReq3, appLr3.getLocalPath(), appLr3.getSize() + 7));
        assertNotNull("Localization not started", pubLr1.getLocalPath());
        pubTracker.handle(new ResourceLocalizedEvent(pubReq1, pubLr1.getLocalPath(), pubLr1.getSize() + 1000));
        assertNotNull("Localization not started", pubLr2.getLocalPath());
        pubTracker.handle(new ResourceLocalizedEvent(pubReq2, pubLr2.getLocalPath(), pubLr2.getSize() + 99999));
        dispatcher.await();
        assertEquals(ResourceState.LOCALIZED, privLr1.getState());
        assertEquals(ResourceState.LOCALIZED, privLr2.getState());
        assertEquals(ResourceState.LOCALIZED, appLr1.getState());
        assertEquals(ResourceState.DOWNLOADING, appLr2.getState());
        assertEquals(ResourceState.LOCALIZED, appLr3.getState());
        assertEquals(ResourceState.LOCALIZED, pubLr1.getState());
        assertEquals(ResourceState.LOCALIZED, pubLr2.getState());
        // restart and recover
        spyService = createSpyService(dispatcher, dirsHandler, stateStore);
        spyService.init(conf);
        spyService.recoverLocalizedResources(stateStore.loadLocalizationState());
        dispatcher.await();
        appTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user1, appId1);
        privTracker1 = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user1, null);
        appTracker2 = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user2, appId2);
        pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, null, null);
        LocalizedResource recoveredRsrc = privTracker1.getLocalizedResource(privReq1);
        assertEquals(privReq1, recoveredRsrc.getRequest());
        assertEquals(privLr1.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(privLr1.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = privTracker1.getLocalizedResource(privReq2);
        assertEquals(privReq2, recoveredRsrc.getRequest());
        assertEquals(privLr2.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(privLr2.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = appTracker1.getLocalizedResource(appReq1);
        assertEquals(appReq1, recoveredRsrc.getRequest());
        assertEquals(appLr1.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(appLr1.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
        recoveredRsrc = appTracker2.getLocalizedResource(appReq2);
        assertNull("in-progress resource should not be present", recoveredRsrc);
        recoveredRsrc = appTracker2.getLocalizedResource(appReq3);
        assertEquals(appReq3, recoveredRsrc.getRequest());
        assertEquals(appLr3.getLocalPath(), recoveredRsrc.getLocalPath());
        assertEquals(appLr3.getSize(), recoveredRsrc.getSize());
        assertEquals(ResourceState.LOCALIZED, recoveredRsrc.getState());
    } finally {
        dispatcher.stop();
        stateStore.close();
    }
}