Examples with Container org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container used on opensource projects

Example 36 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestResourceLocalizationService method testFailedDirsResourceRelease.

/*
   * Test to ensure ResourceLocalizationService can handle local dirs going bad.
   * Test first sets up all the components required, then sends events to fetch
   * a private, app and public resource. It then sends events to clean up the
   * container and the app and ensures the right delete calls were made.
   */
@Test
@SuppressWarnings("unchecked")
public // mocked generics
void testFailedDirsResourceRelease() throws Exception {
    // setup components
    File f = new File(basedir.toString());
    String[] sDirs = new String[4];
    List<Path> localDirs = new ArrayList<Path>(sDirs.length);
    for (int i = 0; i < 4; ++i) {
        sDirs[i] = f.getAbsolutePath() + i;
        localDirs.add(new Path(sDirs[i]));
    }
    List<Path> containerLocalDirs = new ArrayList<Path>(localDirs.size());
    List<Path> appLocalDirs = new ArrayList<Path>(localDirs.size());
    List<Path> nmLocalContainerDirs = new ArrayList<Path>(localDirs.size());
    List<Path> nmLocalAppDirs = new ArrayList<Path>(localDirs.size());
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    conf.setLong(YarnConfiguration.NM_DISK_HEALTH_CHECK_INTERVAL_MS, 500);
    LocalizerTracker mockLocallilzerTracker = mock(LocalizerTracker.class);
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    // Ignore actual localization
    EventHandler<LocalizerEvent> localizerBus = mock(EventHandler.class);
    dispatcher.register(LocalizerEventType.class, localizerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalDirsHandlerService mockDirsHandler = mock(LocalDirsHandlerService.class);
    doReturn(new ArrayList<String>(Arrays.asList(sDirs))).when(mockDirsHandler).getLocalDirsForCleanup();
    DeletionService delService = mock(DeletionService.class);
    // setup mocks
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, mockDirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(mockLocallilzerTracker).when(spyService).createLocalizerTracker(isA(Configuration.class));
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    FsPermission defaultPermission = FsPermission.getDirDefault().applyUMask(lfs.getUMask());
    FsPermission nmPermission = ResourceLocalizationService.NM_PRIVATE_PERM.applyUMask(lfs.getUMask());
    final FileStatus fs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, defaultPermission, "", "", localDirs.get(0));
    final FileStatus nmFs = new FileStatus(0, true, 1, 0, System.currentTimeMillis(), 0, nmPermission, "", "", localDirs.get(0));
    final String user = "user0";
    // init application
    final Application app = mock(Application.class);
    final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
    when(app.getUser()).thenReturn(user);
    when(app.getAppId()).thenReturn(appId);
    when(app.toString()).thenReturn(appId.toString());
    // init container.
    final Container c = getMockContainer(appId, 42, user);
    // setup local app dirs
    List<String> tmpDirs = mockDirsHandler.getLocalDirs();
    for (int i = 0; i < tmpDirs.size(); ++i) {
        Path usersdir = new Path(tmpDirs.get(i), ContainerLocalizer.USERCACHE);
        Path userdir = new Path(usersdir, user);
        Path allAppsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
        Path appDir = new Path(allAppsdir, appId.toString());
        Path containerDir = new Path(appDir, c.getContainerId().toString());
        containerLocalDirs.add(containerDir);
        appLocalDirs.add(appDir);
        Path sysDir = new Path(tmpDirs.get(i), ResourceLocalizationService.NM_PRIVATE_DIR);
        Path appSysDir = new Path(sysDir, appId.toString());
        Path containerSysDir = new Path(appSysDir, c.getContainerId().toString());
        nmLocalContainerDirs.add(containerSysDir);
        nmLocalAppDirs.add(appSysDir);
    }
    try {
        spyService.init(conf);
        spyService.start();
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // Get a handle on the trackers after they're setup with
        // INIT_APP_RESOURCES
        LocalResourcesTracker appTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.APPLICATION, user, appId);
        LocalResourcesTracker privTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId);
        LocalResourcesTracker pubTracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
        // init resources
        Random r = new Random();
        long seed = r.nextLong();
        r.setSeed(seed);
        // Send localization requests, one for each type of resource
        final LocalResource privResource = getPrivateMockedResource(r);
        final LocalResourceRequest privReq = new LocalResourceRequest(privResource);
        final LocalResource appResource = getAppMockedResource(r);
        final LocalResourceRequest appReq = new LocalResourceRequest(appResource);
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq));
        req.put(LocalResourceVisibility.APPLICATION, Collections.singletonList(appReq));
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req2 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req2.put(LocalResourceVisibility.PRIVATE, Collections.singletonList(privReq));
        // Send Request event
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        spyService.handle(new ContainerLocalizationRequestEvent(c, req2));
        dispatcher.await();
        int privRsrcCount = 0;
        for (LocalizedResource lr : privTracker) {
            privRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 2, lr.getRefCount());
            Assert.assertEquals(privReq, lr.getRequest());
        }
        Assert.assertEquals(1, privRsrcCount);
        int appRsrcCount = 0;
        for (LocalizedResource lr : appTracker) {
            appRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            Assert.assertEquals(appReq, lr.getRequest());
        }
        Assert.assertEquals(1, appRsrcCount);
        int pubRsrcCount = 0;
        for (LocalizedResource lr : pubTracker) {
            pubRsrcCount++;
            Assert.assertEquals("Incorrect reference count", 1, lr.getRefCount());
            Assert.assertEquals(pubReq, lr.getRequest());
        }
        Assert.assertEquals(1, pubRsrcCount);
        // go through
        for (int i = 0; i < containerLocalDirs.size(); ++i) {
            if (i == 2) {
                Mockito.doThrow(new IOException()).when(spylfs).getFileStatus(eq(containerLocalDirs.get(i)));
                Mockito.doThrow(new IOException()).when(spylfs).getFileStatus(eq(nmLocalContainerDirs.get(i)));
            } else {
                doReturn(fs).when(spylfs).getFileStatus(eq(containerLocalDirs.get(i)));
                doReturn(nmFs).when(spylfs).getFileStatus(eq(nmLocalContainerDirs.get(i)));
            }
        }
        // Send Cleanup Event
        spyService.handle(new ContainerLocalizationCleanupEvent(c, req));
        verify(mockLocallilzerTracker).cleanupPrivLocalizers("container_314159265358979_0003_01_000042");
        // match cleanup events with the mocks we setup earlier
        for (int i = 0; i < containerLocalDirs.size(); ++i) {
            if (i == 2) {
                try {
                    verify(delService).delete(user, containerLocalDirs.get(i));
                    verify(delService).delete(null, nmLocalContainerDirs.get(i));
                    Assert.fail("deletion attempts for invalid dirs");
                } catch (Throwable e) {
                    continue;
                }
            } else {
                verify(delService).delete(user, containerLocalDirs.get(i));
                verify(delService).delete(null, nmLocalContainerDirs.get(i));
            }
        }
        ArgumentMatcher<ApplicationEvent> matchesAppDestroy = new ArgumentMatcher<ApplicationEvent>() {

            @Override
            public boolean matches(Object o) {
                ApplicationEvent evt = (ApplicationEvent) o;
                return (evt.getType() == ApplicationEventType.APPLICATION_RESOURCES_CLEANEDUP) && appId == evt.getApplicationID();
            }
        };
        dispatcher.await();
        // IOExceptions
        for (int i = 0; i < containerLocalDirs.size(); ++i) {
            if (i == 3) {
                Mockito.doThrow(new IOException()).when(spylfs).getFileStatus(eq(appLocalDirs.get(i)));
                Mockito.doThrow(new UnsupportedFileSystemException("test")).when(spylfs).getFileStatus(eq(nmLocalAppDirs.get(i)));
            } else {
                doReturn(fs).when(spylfs).getFileStatus(eq(appLocalDirs.get(i)));
                doReturn(nmFs).when(spylfs).getFileStatus(eq(nmLocalAppDirs.get(i)));
            }
        }
        LocalizationEvent destroyApp = new ApplicationLocalizationEvent(LocalizationEventType.DESTROY_APPLICATION_RESOURCES, app);
        spyService.handle(destroyApp);
        // Waits for APPLICATION_RESOURCES_CLEANEDUP event to be handled.
        dispatcher.await();
        verify(applicationBus).handle(argThat(matchesAppDestroy));
        // verify we got the right delete calls
        for (int i = 0; i < containerLocalDirs.size(); ++i) {
            if (i == 3) {
                try {
                    verify(delService).delete(user, containerLocalDirs.get(i));
                    verify(delService).delete(null, nmLocalContainerDirs.get(i));
                    Assert.fail("deletion attempts for invalid dirs");
                } catch (Throwable e) {
                    continue;
                }
            } else {
                verify(delService).delete(user, appLocalDirs.get(i));
                verify(delService).delete(null, nmLocalAppDirs.get(i));
            }
        }
    } finally {
        dispatcher.stop();
        delService.stop();
    }
}

Example 37 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestResourceLocalizationService method testPublicResourceAddResourceExceptions.

/*
   * Test case for handling RejectedExecutionException and IOException which can
   * be thrown when adding public resources to the pending queue.
   * RejectedExecutionException can be thrown either due to the incoming queue
   * being full or if the ExecutorCompletionService threadpool is shutdown.
   * Since it's hard to simulate the queue being full, this test just shuts down
   * the threadpool and makes sure the exception is handled. If anything is
   * messed up the async dispatcher thread will cause a system exit causing the
   * test to fail.
   */
@Test
@SuppressWarnings("unchecked")
public void testPublicResourceAddResourceExceptions() throws Exception {
    List<Path> localDirs = new ArrayList<Path>();
    String[] sDirs = new String[4];
    for (int i = 0; i < 4; ++i) {
        localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
        sDirs[i] = localDirs.get(i).toString();
    }
    conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
    conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
    DrainDispatcher dispatcher = new DrainDispatcher();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    DeletionService delService = mock(DeletionService.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
    dirsHandlerSpy.init(conf);
    dispatcher.init(conf);
    dispatcher.start();
    try {
        ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext);
        ResourceLocalizationService spyService = spy(rawService);
        doReturn(mockServer).when(spyService).createServer();
        doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
        spyService.init(conf);
        spyService.start();
        final String user = "user0";
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn(user);
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // init resources
        Random r = new Random();
        r.setSeed(r.nextLong());
        // Queue localization request for the public resource
        final LocalResource pubResource = getPublicMockedResource(r);
        final LocalResourceRequest pubReq = new LocalResourceRequest(pubResource);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq));
        // init container.
        final Container c = getMockContainer(appId, 42, user);
        // first test ioexception
        Mockito.doThrow(new IOException()).when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class), Mockito.anyLong(), Mockito.anyBoolean());
        // send request
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        dispatcher.await();
        LocalResourcesTracker tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
        Assert.assertNull(tracker.getLocalizedResource(pubReq));
        // test IllegalArgumentException
        String name = Long.toHexString(r.nextLong());
        URL url = getPath("/local/PRIVATE/" + name + "/");
        final LocalResource rsrc = BuilderUtils.newLocalResource(url, LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, r.nextInt(1024) + 1024L, r.nextInt(1024) + 2048L, false);
        final LocalResourceRequest pubReq1 = new LocalResourceRequest(rsrc);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req1 = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        req1.put(LocalResourceVisibility.PUBLIC, Collections.singletonList(pubReq1));
        Mockito.doCallRealMethod().when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class), Mockito.anyLong(), Mockito.anyBoolean());
        // send request
        spyService.handle(new ContainerLocalizationRequestEvent(c, req1));
        dispatcher.await();
        tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
        Assert.assertNull(tracker.getLocalizedResource(pubReq));
        // test RejectedExecutionException by shutting down the thread pool
        PublicLocalizer publicLocalizer = spyService.getPublicLocalizer();
        publicLocalizer.threadPool.shutdown();
        spyService.handle(new ContainerLocalizationRequestEvent(c, req));
        dispatcher.await();
        tracker = spyService.getLocalResourcesTracker(LocalResourceVisibility.PUBLIC, user, appId);
        Assert.assertNull(tracker.getLocalizedResource(pubReq));
    } finally {
        // if we call stop with events in the queue, an InterruptedException gets
        // thrown resulting in the dispatcher thread causing a system exit
        dispatcher.await();
        dispatcher.stop();
    }
}

Example 38 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestResourceLocalizationService method getMockContainer.

private static Container getMockContainer(ApplicationId appId, int id, String user) throws IOException {
    Container c = mock(Container.class);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId cId = BuilderUtils.newContainerId(appAttemptId, id);
    when(c.getUser()).thenReturn(user);
    when(c.getContainerId()).thenReturn(cId);
    Credentials creds = new Credentials();
    Token<? extends TokenIdentifier> tk = getToken(id);
    String fingerprint = ResourceLocalizationService.buildTokenFingerprint(tk);
    assertNotNull(fingerprint);
    assertTrue("Expected token fingerprint of 10 hex bytes delimited by space.", fingerprint.matches("^(([0-9a-f]){2} ){9}([0-9a-f]){2}$"));
    creds.addToken(new Text("tok" + id), tk);
    when(c.getCredentials()).thenReturn(creds);
    when(c.toString()).thenReturn(cId.toString());
    when(c.getContainerState()).thenReturn(ContainerState.LOCALIZING);
    return c;
}

Example 39 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestResourceLocalizationService method testLocalResourcePath.

@Test(timeout = 10000)
@SuppressWarnings("unchecked")
public void testLocalResourcePath() throws Exception {
    // test the local path where application and user cache files will be
    // localized.
    DrainDispatcher dispatcher1 = null;
    try {
        dispatcher1 = new DrainDispatcher();
        String user = "testuser";
        ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
        // creating one local directory
        List<Path> localDirs = new ArrayList<Path>();
        String[] sDirs = new String[1];
        for (int i = 0; i < 1; ++i) {
            localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
            sDirs[i] = localDirs.get(i).toString();
        }
        conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
        LocalDirsHandlerService localDirHandler = new LocalDirsHandlerService();
        localDirHandler.init(conf);
        // Registering event handlers
        EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
        dispatcher1.register(ApplicationEventType.class, applicationBus);
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        dispatcher1.register(ContainerEventType.class, containerBus);
        ContainerExecutor exec = mock(ContainerExecutor.class);
        DeletionService delService = mock(DeletionService.class);
        LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
        // initializing directory handler.
        dirsHandler.init(conf);
        dispatcher1.init(conf);
        dispatcher1.start();
        ResourceLocalizationService rls = new ResourceLocalizationService(dispatcher1, exec, delService, localDirHandler, nmContext);
        dispatcher1.register(LocalizationEventType.class, rls);
        rls.init(conf);
        rls.handle(createApplicationLocalizationEvent(user, appId));
        // We need to pre-populate the LocalizerRunner as the
        // Resource Localization Service code internally starts them which
        // definitely we don't want.
        // creating new container and populating corresponding localizer runner
        // Container - 1
        Container container1 = createMockContainer(user, 1);
        String localizerId1 = container1.getContainerId().toString();
        rls.getPrivateLocalizers().put(localizerId1, rls.new LocalizerRunner(new LocalizerContext(user, container1.getContainerId(), null), localizerId1));
        // Creating two requests for container
        // 1) Private resource
        // 2) Application resource
        LocalResourceRequest reqPriv = new LocalResourceRequest(new Path("file:///tmp1"), 123L, LocalResourceType.FILE, LocalResourceVisibility.PRIVATE, "");
        List<LocalResourceRequest> privList = new ArrayList<LocalResourceRequest>();
        privList.add(reqPriv);
        LocalResourceRequest reqApp = new LocalResourceRequest(new Path("file:///tmp2"), 123L, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, "");
        List<LocalResourceRequest> appList = new ArrayList<LocalResourceRequest>();
        appList.add(reqApp);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        rsrcs.put(LocalResourceVisibility.APPLICATION, appList);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privList);
        dispatcher1.getEventHandler().handle(new ContainerLocalizationRequestEvent(container1, rsrcs));
        // Now waiting for resource download to start. Here actual will not start
        // Only the resources will be populated into pending list.
        Assert.assertTrue(waitForPrivateDownloadToStart(rls, localizerId1, 2, 5000));
        // Validating user and application cache paths
        String userCachePath = StringUtils.join(Path.SEPARATOR, Arrays.asList(localDirs.get(0).toUri().getRawPath(), ContainerLocalizer.USERCACHE, user, ContainerLocalizer.FILECACHE));
        String userAppCachePath = StringUtils.join(Path.SEPARATOR, Arrays.asList(localDirs.get(0).toUri().getRawPath(), ContainerLocalizer.USERCACHE, user, ContainerLocalizer.APPCACHE, appId.toString(), ContainerLocalizer.FILECACHE));
        // Now the Application and private resources may come in any order
        // for download.
        // For User cahce :
        // returned destinationPath = user cache path + random number
        // For App cache :
        // returned destinationPath = user app cache path + random number
        int returnedResources = 0;
        boolean appRsrc = false, privRsrc = false;
        while (returnedResources < 2) {
            LocalizerHeartbeatResponse response = rls.heartbeat(createLocalizerStatus(localizerId1));
            for (ResourceLocalizationSpec resourceSpec : response.getResourceSpecs()) {
                returnedResources++;
                Path destinationDirectory = new Path(resourceSpec.getDestinationDirectory().getFile());
                if (resourceSpec.getResource().getVisibility() == LocalResourceVisibility.APPLICATION) {
                    appRsrc = true;
                    Assert.assertEquals(userAppCachePath, destinationDirectory.getParent().toUri().toString());
                } else if (resourceSpec.getResource().getVisibility() == LocalResourceVisibility.PRIVATE) {
                    privRsrc = true;
                    Assert.assertEquals(userCachePath, destinationDirectory.getParent().toUri().toString());
                } else {
                    throw new Exception("Unexpected resource received.");
                }
            }
        }
        // We should receive both the resources (Application and Private)
        Assert.assertTrue(appRsrc && privRsrc);
    } finally {
        if (dispatcher1 != null) {
            dispatcher1.stop();
        }
    }
}

Example 40 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestResourceLocalizationService method testLocalizerRunnerException.

@Test(timeout = 10000)
// mocked generics
@SuppressWarnings("unchecked")
public void testLocalizerRunnerException() throws Exception {
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
    dispatcher.register(ApplicationEventType.class, applicationBus);
    EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
    dispatcher.register(ContainerEventType.class, containerBus);
    ContainerExecutor exec = mock(ContainerExecutor.class);
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    LocalDirsHandlerService dirsHandlerSpy = spy(dirsHandler);
    dirsHandlerSpy.init(conf);
    DeletionService delServiceReal = new DeletionService(exec);
    DeletionService delService = spy(delServiceReal);
    delService.init(new Configuration());
    delService.start();
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, delService, dirsHandlerSpy, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    try {
        spyService.init(conf);
        spyService.start();
        // init application
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(314159265358979L, 3);
        when(app.getUser()).thenReturn("user0");
        when(app.getAppId()).thenReturn(appId);
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c = getMockContainer(appId, 42, "user0");
        final LocalResource resource1 = getPrivateMockedResource(r);
        System.out.println("Here 4");
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> privateResourceList = new ArrayList<LocalResourceRequest>();
        privateResourceList.add(req1);
        rsrcs.put(LocalResourceVisibility.PRIVATE, privateResourceList);
        final Constructor<?>[] constructors = FSError.class.getDeclaredConstructors();
        constructors[0].setAccessible(true);
        FSError fsError = (FSError) constructors[0].newInstance(new IOException("Disk Error"));
        Mockito.doThrow(fsError).when(dirsHandlerSpy).getLocalPathForWrite(isA(String.class));
        spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
        Thread.sleep(1000);
        dispatcher.await();
        // Verify if ContainerResourceFailedEvent is invoked on FSError
        verify(containerBus).handle(isA(ContainerResourceFailedEvent.class));
    } finally {
        spyService.stop();
        dispatcher.stop();
        delService.stop();
    }
}