Examples with LocalDirsHandlerService org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService used on opensource projects

Example 21 with LocalDirsHandlerService

use of org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService in project hadoop by apache.

the class TestAppLogAggregatorImpl method createLocalDirsHandlerService.

private static LocalDirsHandlerService createLocalDirsHandlerService(YarnConfiguration conf, final String rootLogDir) {
    LocalDirsHandlerService dirsHandlerService = new LocalDirsHandlerService() {

        @Override
        public List<String> getLogDirsForRead() {
            return new ArrayList<String>() {

                {
                    add(rootLogDir);
                }
            };
        }

        @Override
        public List<String> getLogDirsForCleanup() {
            return new ArrayList<String>() {

                {
                    add(rootLogDir);
                }
            };
        }
    };
    dirsHandlerService.init(conf);
    // so it is ok to not start the service.
    return dirsHandlerService;
}

Example 22 with LocalDirsHandlerService

use of org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService in project hadoop by apache.

the class TestLocalResourcesTrackerImpl method testResourcePresentInGoodDir.

@SuppressWarnings("unchecked")
@Test
public void testResourcePresentInGoodDir() throws IOException {
    String user = "testuser";
    DrainDispatcher dispatcher = null;
    try {
        Configuration conf = new Configuration();
        dispatcher = createDispatcher(conf);
        EventHandler<LocalizerEvent> localizerEventHandler = mock(EventHandler.class);
        EventHandler<LocalizerEvent> containerEventHandler = mock(EventHandler.class);
        dispatcher.register(LocalizerEventType.class, localizerEventHandler);
        dispatcher.register(ContainerEventType.class, containerEventHandler);
        ContainerId cId1 = BuilderUtils.newContainerId(1, 1, 1, 1);
        LocalizerContext lc1 = new LocalizerContext(user, cId1, null);
        LocalResourceRequest req1 = createLocalResourceRequest(user, 1, 1, LocalResourceVisibility.PUBLIC);
        LocalResourceRequest req2 = createLocalResourceRequest(user, 2, 1, LocalResourceVisibility.PUBLIC);
        LocalizedResource lr1 = createLocalizedResource(req1, dispatcher);
        LocalizedResource lr2 = createLocalizedResource(req2, dispatcher);
        ConcurrentMap<LocalResourceRequest, LocalizedResource> localrsrc = new ConcurrentHashMap<LocalResourceRequest, LocalizedResource>();
        localrsrc.put(req1, lr1);
        localrsrc.put(req2, lr2);
        LocalDirsHandlerService dirsHandler = mock(LocalDirsHandlerService.class);
        List<String> goodDirs = new ArrayList<String>();
        // /tmp/somedir2 is bad
        goodDirs.add("/tmp/somedir1/");
        goodDirs.add("/tmp/somedir2");
        Mockito.when(dirsHandler.getLocalDirs()).thenReturn(goodDirs);
        Mockito.when(dirsHandler.getLocalDirsForRead()).thenReturn(goodDirs);
        LocalResourcesTrackerImpl tracker = new LocalResourcesTrackerImpl(user, null, dispatcher, localrsrc, true, conf, new NMNullStateStoreService(), dirsHandler);
        ResourceEvent req11Event = new ResourceRequestEvent(req1, LocalResourceVisibility.PUBLIC, lc1);
        ResourceEvent req21Event = new ResourceRequestEvent(req2, LocalResourceVisibility.PUBLIC, lc1);
        // Localize R1 for C1
        tracker.handle(req11Event);
        // Localize R2 for C1
        tracker.handle(req21Event);
        dispatcher.await();
        // Localize resource1
        Path p1 = tracker.getPathForLocalization(req1, new Path("/tmp/somedir1"), null);
        Path p2 = tracker.getPathForLocalization(req2, new Path("/tmp/somedir2"), null);
        ResourceLocalizedEvent rle1 = new ResourceLocalizedEvent(req1, p1, 1);
        tracker.handle(rle1);
        ResourceLocalizedEvent rle2 = new ResourceLocalizedEvent(req2, p2, 1);
        tracker.handle(rle2);
        dispatcher.await();
        // Remove somedir2 from gooddirs
        Assert.assertTrue(tracker.checkLocalResource(lr2));
        goodDirs.remove(1);
        Assert.assertFalse(tracker.checkLocalResource(lr2));
    } finally {
        if (dispatcher != null) {
            dispatcher.stop();
        }
    }
}

Example 23 with LocalDirsHandlerService

use of org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService in project hadoop by apache.

the class TestContainerManagerRecovery method setup.

@Override
@Before
public void setup() throws IOException {
    localFS.delete(new Path(localDir.getAbsolutePath()), true);
    localFS.delete(new Path(tmpDir.getAbsolutePath()), true);
    localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
    localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
    localDir.mkdir();
    tmpDir.mkdir();
    localLogDir.mkdir();
    remoteLogDir.mkdir();
    LOG.info("Created localDir in " + localDir.getAbsolutePath());
    LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());
    String bindAddress = "0.0.0.0:" + ServerSocketUtil.getPort(49160, 10);
    conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());
    conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
    // Default delSrvc
    delSrvc = createDeletionService();
    delSrvc.init(conf);
    exec = createContainerExecutor();
    dirsHandler = new LocalDirsHandlerService();
    nodeHealthChecker = new NodeHealthCheckerService(NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
    nodeHealthChecker.init(conf);
}

Example 24 with LocalDirsHandlerService

use of org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService in project hadoop by apache.

the class BaseContainerManagerTest method setup.

@Before
public void setup() throws IOException {
    localFS.delete(new Path(localDir.getAbsolutePath()), true);
    localFS.delete(new Path(tmpDir.getAbsolutePath()), true);
    localFS.delete(new Path(localLogDir.getAbsolutePath()), true);
    localFS.delete(new Path(remoteLogDir.getAbsolutePath()), true);
    localDir.mkdir();
    tmpDir.mkdir();
    localLogDir.mkdir();
    remoteLogDir.mkdir();
    LOG.info("Created localDir in " + localDir.getAbsolutePath());
    LOG.info("Created tmpDir in " + tmpDir.getAbsolutePath());
    String bindAddress = "0.0.0.0:" + ServerSocketUtil.getPort(49162, 10);
    conf.set(YarnConfiguration.NM_ADDRESS, bindAddress);
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, localLogDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_REMOTE_APP_LOG_DIR, remoteLogDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOCALIZER_ADDRESS, "0.0.0.0:" + ServerSocketUtil.getPort(8040, 10));
    conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
    // Default delSrvc
    exec = createContainerExecutor();
    delSrvc = createDeletionService();
    delSrvc.init(conf);
    dirsHandler = new LocalDirsHandlerService();
    nodeHealthChecker = new NodeHealthCheckerService(NodeManager.getNodeHealthScriptRunner(conf), dirsHandler);
    nodeHealthChecker.init(conf);
    containerManager = createContainerManager(delSrvc);
    ((NMContext) context).setContainerManager(containerManager);
    nodeStatusUpdater.init(conf);
    containerManager.init(conf);
    nodeStatusUpdater.start();
    ((NMContext) context).setNodeStatusUpdater(nodeStatusUpdater);
}

Example 25 with LocalDirsHandlerService

use of org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService in project hadoop by apache.

the class TestResourceLocalizationService method testLocalizerHeartbeatWhenAppCleaningUp.

@Test(timeout = 20000)
@SuppressWarnings("unchecked")
public void testLocalizerHeartbeatWhenAppCleaningUp() throws Exception {
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, lfs.makeQualified(new Path(basedir, 0 + "")).toString());
    // Start dispatcher.
    DrainDispatcher dispatcher = new DrainDispatcher();
    dispatcher.init(conf);
    dispatcher.start();
    dispatcher.register(ApplicationEventType.class, mock(EventHandler.class));
    dispatcher.register(ContainerEventType.class, mock(EventHandler.class));
    DummyExecutor exec = new DummyExecutor();
    LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    // Start resource localization service.
    ResourceLocalizationService rawService = new ResourceLocalizationService(dispatcher, exec, mock(DeletionService.class), dirsHandler, nmContext);
    ResourceLocalizationService spyService = spy(rawService);
    doReturn(mockServer).when(spyService).createServer();
    doReturn(lfs).when(spyService).getLocalFileContext(isA(Configuration.class));
    try {
        spyService.init(conf);
        spyService.start();
        // Init application resources.
        final Application app = mock(Application.class);
        final ApplicationId appId = BuilderUtils.newApplicationId(1234567890L, 3);
        when(app.getUser()).thenReturn("user0");
        when(app.getAppId()).thenReturn(appId);
        when(app.toString()).thenReturn(appId.toString());
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.INIT_APPLICATION_RESOURCES, app));
        dispatcher.await();
        // Initialize localizer.
        Random r = new Random();
        long seed = r.nextLong();
        System.out.println("SEED: " + seed);
        r.setSeed(seed);
        final Container c = getMockContainer(appId, 46, "user0");
        FSDataOutputStream out = new FSDataOutputStream(new DataOutputBuffer(), null);
        doReturn(out).when(spylfs).createInternal(isA(Path.class), isA(EnumSet.class), isA(FsPermission.class), anyInt(), anyShort(), anyLong(), isA(Progressable.class), isA(ChecksumOpt.class), anyBoolean());
        final LocalResource resource1 = getAppMockedResource(r);
        final LocalResource resource2 = getAppMockedResource(r);
        // Send localization requests for container.
        // 2 resources generated with APPLICATION visibility.
        final LocalResourceRequest req1 = new LocalResourceRequest(resource1);
        final LocalResourceRequest req2 = new LocalResourceRequest(resource2);
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> rsrcs = new HashMap<LocalResourceVisibility, Collection<LocalResourceRequest>>();
        List<LocalResourceRequest> appResourceList = Arrays.asList(req1, req2);
        rsrcs.put(LocalResourceVisibility.APPLICATION, appResourceList);
        spyService.handle(new ContainerLocalizationRequestEvent(c, rsrcs));
        dispatcher.await();
        // Wait for localization to begin.
        exec.waitForLocalizers(1);
        final String containerIdStr = c.getContainerId().toString();
        LocalizerRunner locRunnerForContainer = spyService.getLocalizerRunner(containerIdStr);
        // Heartbeats from container localizer
        LocalResourceStatus rsrcSuccess = mock(LocalResourceStatus.class);
        LocalizerStatus stat = mock(LocalizerStatus.class);
        when(stat.getLocalizerId()).thenReturn(containerIdStr);
        when(rsrcSuccess.getResource()).thenReturn(resource1);
        when(rsrcSuccess.getLocalSize()).thenReturn(4344L);
        when(rsrcSuccess.getLocalPath()).thenReturn(getPath("/some/path"));
        when(rsrcSuccess.getStatus()).thenReturn(ResourceStatusType.FETCH_SUCCESS);
        when(stat.getResources()).thenReturn(Collections.<LocalResourceStatus>emptyList());
        // First heartbeat which schedules first resource.
        LocalizerHeartbeatResponse response = spyService.heartbeat(stat);
        assertEquals("NM should tell localizer to be LIVE in Heartbeat.", LocalizerAction.LIVE, response.getLocalizerAction());
        // Cleanup application.
        spyService.handle(new ContainerLocalizationCleanupEvent(c, rsrcs));
        spyService.handle(new ApplicationLocalizationEvent(LocalizationEventType.DESTROY_APPLICATION_RESOURCES, app));
        dispatcher.await();
        try {
            // Directly send heartbeat to introduce race as app is being cleaned up.
            locRunnerForContainer.processHeartbeat(Collections.singletonList(rsrcSuccess));
        } catch (Exception e) {
            fail("Exception should not have been thrown on processing heartbeat");
        }
        // Send another heartbeat.
        response = spyService.heartbeat(stat);
        assertEquals("NM should tell localizer to DIE in Heartbeat.", LocalizerAction.DIE, response.getLocalizerAction());
        exec.setStopLocalization();
    } finally {
        spyService.stop();
        dispatcher.stop();
    }
}