Examples with Container org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container used on opensource projects

Example 86 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestNodeStatusUpdater method testRecentlyFinishedContainers.

@Test(timeout = 90000)
public void testRecentlyFinishedContainers() throws Exception {
    NodeManager nm = new NodeManager();
    YarnConfiguration conf = new YarnConfiguration();
    conf.set(NodeStatusUpdaterImpl.YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS, "10000");
    nm.init(conf);
    NodeStatusUpdaterImpl nodeStatusUpdater = (NodeStatusUpdaterImpl) nm.getNodeStatusUpdater();
    ApplicationId appId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 0);
    ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
    nm.getNMContext().getApplications().putIfAbsent(appId, mock(Application.class));
    nm.getNMContext().getContainers().putIfAbsent(cId, mock(Container.class));
    nodeStatusUpdater.addCompletedContainer(cId);
    Assert.assertTrue(nodeStatusUpdater.isContainerRecentlyStopped(cId));
    nm.getNMContext().getContainers().remove(cId);
    long time1 = System.currentTimeMillis();
    int waitInterval = 15;
    while (waitInterval-- > 0 && nodeStatusUpdater.isContainerRecentlyStopped(cId)) {
        nodeStatusUpdater.removeVeryOldStoppedContainersFromCache();
        Thread.sleep(1000);
    }
    long time2 = System.currentTimeMillis();
    // By this time the container will be removed from cache. need to verify.
    Assert.assertFalse(nodeStatusUpdater.isContainerRecentlyStopped(cId));
    Assert.assertTrue((time2 - time1) >= 10000 && (time2 - time1) <= 250000);
}

Example 87 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestLinuxContainerExecutorWithMocks method testContainerLaunchError.

@Test
public void testContainerLaunchError() throws IOException, ContainerExecutionException {
    // reinitialize executer
    Configuration conf = new Configuration();
    setupMockExecutor(MOCK_EXECUTOR_WITH_ERROR, conf);
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, "file:///bin/echo");
    conf.set(YarnConfiguration.NM_LOG_DIRS, "file:///dev/null");
    LinuxContainerExecutor exec;
    LinuxContainerRuntime linuxContainerRuntime = new DefaultLinuxContainerRuntime(PrivilegedOperationExecutor.getInstance(conf));
    linuxContainerRuntime.initialize(conf);
    exec = new LinuxContainerExecutor(linuxContainerRuntime);
    mockExec = spy(exec);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            String diagnostics = (String) invocationOnMock.getArguments()[0];
            assertTrue("Invalid Diagnostics message: " + diagnostics, diagnostics.contains("badcommand"));
            return null;
        }
    }).when(mockExec).logOutput(any(String.class));
    dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    mockExec.setConf(conf);
    String appSubmitter = "nobody";
    String cmd = String.valueOf(PrivilegedOperation.RunAsUserCommand.LAUNCH_CONTAINER.getValue());
    String appId = "APP_ID";
    String containerId = "CONTAINER_ID";
    Container container = mock(Container.class);
    ContainerId cId = mock(ContainerId.class);
    ContainerLaunchContext context = mock(ContainerLaunchContext.class);
    HashMap<String, String> env = new HashMap<String, String>();
    when(container.getContainerId()).thenReturn(cId);
    when(container.getLaunchContext()).thenReturn(context);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ContainerDiagnosticsUpdateEvent event = (ContainerDiagnosticsUpdateEvent) invocationOnMock.getArguments()[0];
            assertTrue("Invalid Diagnostics message: " + event.getDiagnosticsUpdate(), event.getDiagnosticsUpdate().contains("badcommand"));
            return null;
        }
    }).when(container).handle(any(ContainerDiagnosticsUpdateEvent.class));
    when(cId.toString()).thenReturn(containerId);
    when(context.getEnvironment()).thenReturn(env);
    Path scriptPath = new Path("file:///bin/echo");
    Path tokensPath = new Path("file:///dev/null");
    Path workDir = new Path("/tmp");
    Path pidFile = new Path(workDir, "pid.txt");
    mockExec.activateContainer(cId, pidFile);
    int ret = mockExec.launchContainer(new ContainerStartContext.Builder().setContainer(container).setNmPrivateContainerScriptPath(scriptPath).setNmPrivateTokensPath(tokensPath).setUser(appSubmitter).setAppId(appId).setContainerWorkDir(workDir).setLocalDirs(dirsHandler.getLocalDirs()).setLogDirs(dirsHandler.getLogDirs()).setFilecacheDirs(new ArrayList<>()).setUserLocalDirs(new ArrayList<>()).setContainerLocalDirs(new ArrayList<>()).setContainerLogDirs(new ArrayList<>()).build());
    Assert.assertNotSame(0, ret);
    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, appId, containerId, workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, dirsHandler.getLocalDirs()), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, dirsHandler.getLogDirs()), "cgroups=none"), readMockParams());
}

Example 88 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class NMWebServices method getNodeContainer.

@GET
@Path("/containers/{containerid}")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public ContainerInfo getNodeContainer(@javax.ws.rs.core.Context HttpServletRequest hsr, @PathParam("containerid") String id) {
    ContainerId containerId = null;
    init();
    try {
        containerId = ContainerId.fromString(id);
    } catch (Exception e) {
        throw new BadRequestException("invalid container id, " + id);
    }
    Container container = nmContext.getContainers().get(containerId);
    if (container == null) {
        throw new NotFoundException("container with id, " + id + ", not found");
    }
    return new ContainerInfo(this.nmContext, container, uriInfo.getBaseUri().toString(), webapp.name(), hsr.getRemoteUser());
}

Example 89 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestLinuxContainerExecutor method testPostExecuteAfterReacquisition.

@Test(timeout = 10000)
public void testPostExecuteAfterReacquisition() throws Exception {
    Assume.assumeTrue(shouldRun());
    // make up some bogus container ID
    ApplicationId appId = ApplicationId.newInstance(12345, 67890);
    ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 54321);
    ContainerId cid = ContainerId.newContainerId(attemptId, 9876);
    Configuration conf = new YarnConfiguration();
    conf.setClass(YarnConfiguration.NM_LINUX_CONTAINER_RESOURCES_HANDLER, TestResourceHandler.class, LCEResourcesHandler.class);
    LinuxContainerExecutor lce = new LinuxContainerExecutor();
    lce.setConf(conf);
    try {
        lce.init();
    } catch (IOException e) {
    // expected if LCE isn't setup right, but not necessary for this test
    }
    Container container = mock(Container.class);
    ContainerLaunchContext context = mock(ContainerLaunchContext.class);
    HashMap<String, String> env = new HashMap<>();
    when(container.getLaunchContext()).thenReturn(context);
    when(context.getEnvironment()).thenReturn(env);
    lce.reacquireContainer(new ContainerReacquisitionContext.Builder().setContainer(container).setUser("foouser").setContainerId(cid).build());
    assertTrue("postExec not called after reacquisition", TestResourceHandler.postExecContainers.contains(cid));
}

Example 90 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestContainerManager method testUnauthorizedRequests.

@Test
public void testUnauthorizedRequests() throws IOException, YarnException {
    containerManager.start();
    // Create a containerId that belongs to an unauthorized appId
    ContainerId cId = createContainerId(0, 1);
    // startContainers()
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, createContainerToken(cId, DUMMY_RM_IDENTIFIER, context.getNodeId(), user, context.getContainerTokenSecretManager()));
    List<StartContainerRequest> list = new ArrayList<>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    StartContainersResponse startResponse = containerManager.startContainers(allRequests);
    Assert.assertFalse("Should not be authorized to start container", startResponse.getSuccessfullyStartedContainers().contains(cId));
    Assert.assertTrue("Start container request should fail", startResponse.getFailedRequests().containsKey(cId));
    // Insert the containerId into context, make it as if it is running
    ContainerTokenIdentifier containerTokenIdentifier = BuilderUtils.newContainerTokenIdentifier(scRequest.getContainerToken());
    Container container = new ContainerImpl(conf, null, containerLaunchContext, null, metrics, containerTokenIdentifier, context);
    context.getContainers().put(cId, container);
    // stopContainers()
    List<ContainerId> containerIds = new ArrayList<>();
    containerIds.add(cId);
    StopContainersRequest stopRequest = StopContainersRequest.newInstance(containerIds);
    StopContainersResponse stopResponse = containerManager.stopContainers(stopRequest);
    Assert.assertFalse("Should not be authorized to stop container", stopResponse.getSuccessfullyStoppedContainers().contains(cId));
    Assert.assertTrue("Stop container request should fail", stopResponse.getFailedRequests().containsKey(cId));
    // getContainerStatuses()
    containerIds = new ArrayList<>();
    containerIds.add(cId);
    GetContainerStatusesRequest request = GetContainerStatusesRequest.newInstance(containerIds);
    GetContainerStatusesResponse response = containerManager.getContainerStatuses(request);
    Assert.assertEquals("Should not be authorized to get container status", response.getContainerStatuses().size(), 0);
    Assert.assertTrue("Get status request should fail", response.getFailedRequests().containsKey(cId));
}