Examples with ContainerImpl org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl used on opensource projects

Example 11 with ContainerImpl

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl in project hadoop by apache.

the class TestResourceLocalizationService method testParallelDownloadAttemptsForPrivateResource.

@Test(timeout = 100000)
@SuppressWarnings("unchecked")
public void testParallelDownloadAttemptsForPrivateResource() throws Exception {
    DrainDispatcher dispatcher1 = null;
    try {
        dispatcher1 = new DrainDispatcher();
        String user = "testuser";
        ApplicationId appId = BuilderUtils.newApplicationId(1, 1);
        // creating one local directory
        List<Path> localDirs = new ArrayList<Path>();
        String[] sDirs = new String[1];
        for (int i = 0; i < 1; ++i) {
            localDirs.add(lfs.makeQualified(new Path(basedir, i + "")));
            sDirs[i] = localDirs.get(i).toString();
        }
        conf.setStrings(YarnConfiguration.NM_LOCAL_DIRS, sDirs);
        LocalDirsHandlerService localDirHandler = new LocalDirsHandlerService();
        localDirHandler.init(conf);
        // Registering event handlers
        EventHandler<ApplicationEvent> applicationBus = mock(EventHandler.class);
        dispatcher1.register(ApplicationEventType.class, applicationBus);
        EventHandler<ContainerEvent> containerBus = mock(EventHandler.class);
        dispatcher1.register(ContainerEventType.class, containerBus);
        ContainerExecutor exec = mock(ContainerExecutor.class);
        DeletionService delService = mock(DeletionService.class);
        LocalDirsHandlerService dirsHandler = new LocalDirsHandlerService();
        // initializing directory handler.
        dirsHandler.init(conf);
        dispatcher1.init(conf);
        dispatcher1.start();
        ResourceLocalizationService rls = new ResourceLocalizationService(dispatcher1, exec, delService, localDirHandler, nmContext);
        dispatcher1.register(LocalizationEventType.class, rls);
        rls.init(conf);
        rls.handle(createApplicationLocalizationEvent(user, appId));
        LocalResourceRequest req = new LocalResourceRequest(new Path("file:///tmp"), 123L, LocalResourceType.FILE, LocalResourceVisibility.PRIVATE, "");
        // We need to pre-populate the LocalizerRunner as the
        // Resource Localization Service code internally starts them which
        // definitely we don't want.
        // creating new containers and populating corresponding localizer runners
        // Container - 1
        Container container1 = createMockContainer(user, 1);
        String localizerId1 = container1.getContainerId().toString();
        rls.getPrivateLocalizers().put(localizerId1, rls.new LocalizerRunner(new LocalizerContext(user, container1.getContainerId(), null), localizerId1));
        LocalizerRunner localizerRunner1 = rls.getLocalizerRunner(localizerId1);
        dispatcher1.getEventHandler().handle(createContainerLocalizationEvent(container1, LocalResourceVisibility.PRIVATE, req));
        Assert.assertTrue(waitForPrivateDownloadToStart(rls, localizerId1, 1, 5000));
        // Container - 2 now makes the request.
        ContainerImpl container2 = createMockContainer(user, 2);
        String localizerId2 = container2.getContainerId().toString();
        rls.getPrivateLocalizers().put(localizerId2, rls.new LocalizerRunner(new LocalizerContext(user, container2.getContainerId(), null), localizerId2));
        LocalizerRunner localizerRunner2 = rls.getLocalizerRunner(localizerId2);
        dispatcher1.getEventHandler().handle(createContainerLocalizationEvent(container2, LocalResourceVisibility.PRIVATE, req));
        Assert.assertTrue(waitForPrivateDownloadToStart(rls, localizerId2, 1, 5000));
        // Retrieving localized resource.
        LocalResourcesTracker tracker = rls.getLocalResourcesTracker(LocalResourceVisibility.PRIVATE, user, appId);
        LocalizedResource lr = tracker.getLocalizedResource(req);
        // Resource would now have moved into DOWNLOADING state
        Assert.assertEquals(ResourceState.DOWNLOADING, lr.getState());
        // Resource should have one permit
        Assert.assertEquals(1, lr.sem.availablePermits());
        // Resource Localization Service receives first heart beat from
        // ContainerLocalizer for container1
        LocalizerHeartbeatResponse response1 = rls.heartbeat(createLocalizerStatus(localizerId1));
        // Resource must have been added to scheduled map
        Assert.assertEquals(1, localizerRunner1.scheduled.size());
        // Checking resource in the response and also available permits for it.
        Assert.assertEquals(req.getResource(), response1.getResourceSpecs().get(0).getResource().getResource());
        Assert.assertEquals(0, lr.sem.availablePermits());
        // Resource Localization Service now receives first heart beat from
        // ContainerLocalizer for container2
        LocalizerHeartbeatResponse response2 = rls.heartbeat(createLocalizerStatus(localizerId2));
        // Resource must not have been added to scheduled map
        Assert.assertEquals(0, localizerRunner2.scheduled.size());
        // No resource is returned in response
        Assert.assertEquals(0, response2.getResourceSpecs().size());
        // ContainerLocalizer - 1 now sends failed resource heartbeat.
        rls.heartbeat(createLocalizerStatusForFailedResource(localizerId1, req));
        // Resource Localization should fail and state is modified accordingly.
        // Also Local should be release on the LocalizedResource.
        Assert.assertTrue(waitForResourceState(lr, rls, req, LocalResourceVisibility.PRIVATE, user, appId, ResourceState.FAILED, 5000));
        Assert.assertTrue(lr.getState().equals(ResourceState.FAILED));
        Assert.assertEquals(0, localizerRunner1.scheduled.size());
        // Now Container-2 once again sends heart beat to resource localization
        // service
        // Now container-2 again try to download the resource it should still
        // not get the resource as the resource is now not in DOWNLOADING state.
        response2 = rls.heartbeat(createLocalizerStatus(localizerId2));
        // Resource must not have been added to scheduled map.
        // Also as the resource has failed download it will be removed from
        // pending list.
        Assert.assertEquals(0, localizerRunner2.scheduled.size());
        Assert.assertEquals(0, localizerRunner2.pending.size());
        Assert.assertEquals(0, response2.getResourceSpecs().size());
    } finally {
        if (dispatcher1 != null) {
            dispatcher1.stop();
        }
    }
}

Example 12 with ContainerImpl

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl in project hadoop by apache.

the class TestNMWebServer method testNMWebApp.

@Test
public void testNMWebApp() throws IOException, YarnException {
    Configuration conf = new Configuration();
    Context nmContext = new NodeManager.NMContext(null, null, null, null, null, false, conf);
    ResourceView resourceView = new ResourceView() {

        @Override
        public long getVmemAllocatedForContainers() {
            return 0;
        }

        @Override
        public long getPmemAllocatedForContainers() {
            return 0;
        }

        @Override
        public long getVCoresAllocatedForContainers() {
            return 0;
        }

        @Override
        public boolean isVmemCheckEnabled() {
            return true;
        }

        @Override
        public boolean isPmemCheckEnabled() {
            return true;
        }
    };
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, testRootDir.getAbsolutePath());
    conf.set(YarnConfiguration.NM_LOG_DIRS, testLogDir.getAbsolutePath());
    NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
    healthChecker.init(conf);
    LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
    WebServer server = new WebServer(nmContext, resourceView, new ApplicationACLsManager(conf), dirsHandler);
    server.init(conf);
    server.start();
    // Add an application and the corresponding containers
    RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
    Dispatcher dispatcher = new AsyncDispatcher();
    String user = "nobody";
    long clusterTimeStamp = 1234;
    ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
    Application app = mock(Application.class);
    when(app.getUser()).thenReturn(user);
    when(app.getAppId()).thenReturn(appId);
    nmContext.getApplications().put(appId, app);
    ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
    ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
    ContainerId container2 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 1);
    NodeManagerMetrics metrics = mock(NodeManagerMetrics.class);
    NMStateStoreService stateStore = new NMNullStateStoreService();
    for (ContainerId containerId : new ContainerId[] { container1, container2 }) {
        // TODO: Use builder utils
        ContainerLaunchContext launchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
        long currentTime = System.currentTimeMillis();
        Token containerToken = BuilderUtils.newContainerToken(containerId, 0, "127.0.0.1", 1234, user, BuilderUtils.newResource(1024, 1), currentTime + 10000L, 123, "password".getBytes(), currentTime);
        Context context = mock(Context.class);
        Container container = new ContainerImpl(conf, dispatcher, launchContext, null, metrics, BuilderUtils.newContainerTokenIdentifier(containerToken), context) {

            @Override
            public ContainerState getContainerState() {
                return ContainerState.RUNNING;
            }

            ;
        };
        nmContext.getContainers().put(containerId, container);
        //TODO: Gross hack. Fix in code.
        ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
        nmContext.getApplications().get(applicationId).getContainers().put(containerId, container);
        writeContainerLogs(nmContext, containerId, dirsHandler);
    }
// TODO: Pull logs and test contents.
//    Thread.sleep(1000000);
}

Example 13 with ContainerImpl

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerImpl in project hadoop by apache.

the class TestContainerLogsPage method testLogFileWithDriveLetter.

@Test
public void testLogFileWithDriveLetter() throws Exception {
    ContainerImpl container = mock(ContainerImpl.class);
    ApplicationIdPBImpl appId = mock(ApplicationIdPBImpl.class);
    when(appId.toString()).thenReturn("appId");
    Application app = mock(Application.class);
    when(app.getAppId()).thenReturn(appId);
    ApplicationAttemptIdPBImpl appAttemptId = mock(ApplicationAttemptIdPBImpl.class);
    when(appAttemptId.getApplicationId()).thenReturn(appId);
    ConcurrentMap<ApplicationId, Application> applications = new ConcurrentHashMap<ApplicationId, Application>();
    applications.put(appId, app);
    ContainerId containerId = mock(ContainerIdPBImpl.class);
    when(containerId.toString()).thenReturn("containerId");
    when(containerId.getApplicationAttemptId()).thenReturn(appAttemptId);
    ConcurrentMap<ContainerId, Container> containers = new ConcurrentHashMap<ContainerId, Container>();
    containers.put(containerId, container);
    LocalDirsHandlerService localDirs = mock(LocalDirsHandlerService.class);
    when(localDirs.getLogPathToRead("appId" + Path.SEPARATOR + "containerId" + Path.SEPARATOR + "fileName")).thenReturn(new Path("F:/nmlogs/appId/containerId/fileName"));
    NMContext context = mock(NMContext.class);
    when(context.getLocalDirsHandler()).thenReturn(localDirs);
    when(context.getApplications()).thenReturn(applications);
    when(context.getContainers()).thenReturn(containers);
    File logFile = ContainerLogsUtils.getContainerLogFile(containerId, "fileName", null, context);
    Assert.assertTrue("logFile lost drive letter " + logFile, logFile.toString().indexOf("F:" + File.separator + "nmlogs") > -1);
}