Examples with Container org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container used on opensource projects

Example 1 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestDefaultContainerExecutor method testContainerLaunchError.

@Test
public void testContainerLaunchError() throws IOException, InterruptedException {
    if (Shell.WINDOWS) {
        BASE_TMP_PATH = new Path(new File("target").getAbsolutePath(), TestDefaultContainerExecutor.class.getSimpleName());
    }
    Path localDir = new Path(BASE_TMP_PATH, "localDir");
    List<String> localDirs = new ArrayList<String>();
    localDirs.add(localDir.toString());
    List<String> logDirs = new ArrayList<String>();
    Path logDir = new Path(BASE_TMP_PATH, "logDir");
    logDirs.add(logDir.toString());
    Configuration conf = new Configuration();
    conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, localDir.toString());
    conf.set(YarnConfiguration.NM_LOG_DIRS, logDir.toString());
    FileContext lfs = FileContext.getLocalFSFileContext(conf);
    DefaultContainerExecutor mockExec = spy(new DefaultContainerExecutor(lfs));
    mockExec.setConf(conf);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            String diagnostics = (String) invocationOnMock.getArguments()[0];
            assertTrue("Invalid Diagnostics message: " + diagnostics, diagnostics.contains("No such file or directory"));
            return null;
        }
    }).when(mockExec).logOutput(any(String.class));
    String appSubmitter = "nobody";
    String appId = "APP_ID";
    String containerId = "CONTAINER_ID";
    Container container = mock(Container.class);
    ContainerId cId = mock(ContainerId.class);
    ContainerLaunchContext context = mock(ContainerLaunchContext.class);
    HashMap<String, String> env = new HashMap<String, String>();
    env.put("LANG", "C");
    when(container.getContainerId()).thenReturn(cId);
    when(container.getLaunchContext()).thenReturn(context);
    try {
        doAnswer(new Answer() {

            @Override
            public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
                ContainerDiagnosticsUpdateEvent event = (ContainerDiagnosticsUpdateEvent) invocationOnMock.getArguments()[0];
                assertTrue("Invalid Diagnostics message: " + event.getDiagnosticsUpdate(), event.getDiagnosticsUpdate().contains("No such file or directory"));
                return null;
            }
        }).when(container).handle(any(ContainerDiagnosticsUpdateEvent.class));
        when(cId.toString()).thenReturn(containerId);
        when(cId.getApplicationAttemptId()).thenReturn(ApplicationAttemptId.newInstance(ApplicationId.newInstance(0, 1), 0));
        when(context.getEnvironment()).thenReturn(env);
        mockExec.createUserLocalDirs(localDirs, appSubmitter);
        mockExec.createUserCacheDirs(localDirs, appSubmitter);
        mockExec.createAppDirs(localDirs, appSubmitter, appId);
        mockExec.createAppLogDirs(appId, logDirs, appSubmitter);
        Path scriptPath = new Path("file:///bin/echo");
        Path tokensPath = new Path("file:///dev/null");
        if (Shell.WINDOWS) {
            File tmp = new File(BASE_TMP_PATH.toString(), "test_echo.cmd");
            BufferedWriter output = new BufferedWriter(new FileWriter(tmp));
            output.write("Exit 1");
            output.write("Echo No such file or directory 1>&2");
            output.close();
            scriptPath = new Path(tmp.getAbsolutePath());
            tmp = new File(BASE_TMP_PATH.toString(), "tokens");
            tmp.createNewFile();
            tokensPath = new Path(tmp.getAbsolutePath());
        }
        Path workDir = localDir;
        Path pidFile = new Path(workDir, "pid.txt");
        mockExec.init();
        mockExec.activateContainer(cId, pidFile);
        int ret = mockExec.launchContainer(new ContainerStartContext.Builder().setContainer(container).setNmPrivateContainerScriptPath(scriptPath).setNmPrivateTokensPath(tokensPath).setUser(appSubmitter).setAppId(appId).setContainerWorkDir(workDir).setLocalDirs(localDirs).setLogDirs(logDirs).build());
        Assert.assertNotSame(0, ret);
    } finally {
        mockExec.deleteAsUser(new DeletionAsUserContext.Builder().setUser(appSubmitter).setSubDir(localDir).build());
        mockExec.deleteAsUser(new DeletionAsUserContext.Builder().setUser(appSubmitter).setSubDir(logDir).build());
    }
}

Example 2 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestLinuxContainerExecutorWithMocks method testContainerLaunch.

@Test
public void testContainerLaunch() throws IOException {
    String appSubmitter = "nobody";
    String cmd = String.valueOf(PrivilegedOperation.RunAsUserCommand.LAUNCH_CONTAINER.getValue());
    String appId = "APP_ID";
    String containerId = "CONTAINER_ID";
    Container container = mock(Container.class);
    ContainerId cId = mock(ContainerId.class);
    ContainerLaunchContext context = mock(ContainerLaunchContext.class);
    HashMap<String, String> env = new HashMap<String, String>();
    when(container.getContainerId()).thenReturn(cId);
    when(container.getLaunchContext()).thenReturn(context);
    when(cId.toString()).thenReturn(containerId);
    when(context.getEnvironment()).thenReturn(env);
    Path scriptPath = new Path("file:///bin/echo");
    Path tokensPath = new Path("file:///dev/null");
    Path workDir = new Path("/tmp");
    Path pidFile = new Path(workDir, "pid.txt");
    mockExec.activateContainer(cId, pidFile);
    int ret = mockExec.launchContainer(new ContainerStartContext.Builder().setContainer(container).setNmPrivateContainerScriptPath(scriptPath).setNmPrivateTokensPath(tokensPath).setUser(appSubmitter).setAppId(appId).setContainerWorkDir(workDir).setLocalDirs(dirsHandler.getLocalDirs()).setLogDirs(dirsHandler.getLogDirs()).setFilecacheDirs(new ArrayList<>()).setUserLocalDirs(new ArrayList<>()).setContainerLocalDirs(new ArrayList<>()).setContainerLogDirs(new ArrayList<>()).build());
    assertEquals(0, ret);
    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, appId, containerId, workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, dirsHandler.getLocalDirs()), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, dirsHandler.getLogDirs()), "cgroups=none"), readMockParams());
}

Example 3 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestLinuxContainerExecutorWithMocks method testContainerKill.

@Test
public void testContainerKill() throws IOException {
    String appSubmitter = "nobody";
    String cmd = String.valueOf(PrivilegedOperation.RunAsUserCommand.SIGNAL_CONTAINER.getValue());
    ContainerExecutor.Signal signal = ContainerExecutor.Signal.QUIT;
    String sigVal = String.valueOf(signal.getValue());
    Container container = mock(Container.class);
    ContainerId cId = mock(ContainerId.class);
    ContainerLaunchContext context = mock(ContainerLaunchContext.class);
    when(container.getContainerId()).thenReturn(cId);
    when(container.getLaunchContext()).thenReturn(context);
    mockExec.signalContainer(new ContainerSignalContext.Builder().setContainer(container).setUser(appSubmitter).setPid("1000").setSignal(signal).build());
    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, "1000", sigVal), readMockParams());
}

Example 4 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class NMTimelinePublisher method publishContainerCreatedEvent.

@SuppressWarnings("unchecked")
private void publishContainerCreatedEvent(ContainerEvent event) {
    ContainerId containerId = event.getContainerID();
    ContainerEntity entity = createContainerEntity(containerId);
    Container container = context.getContainers().get(containerId);
    Resource resource = container.getResource();
    Map<String, Object> entityInfo = new HashMap<String, Object>();
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_MEMORY_INFO, resource.getMemorySize());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_VCORE_INFO, resource.getVirtualCores());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_INFO, nodeId.getHost());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_PORT_INFO, nodeId.getPort());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_PRIORITY_INFO, container.getPriority().toString());
    entityInfo.put(ContainerMetricsConstants.ALLOCATED_HOST_HTTP_ADDRESS_INFO, httpAddress);
    entity.setInfo(entityInfo);
    TimelineEvent tEvent = new TimelineEvent();
    tEvent.setId(ContainerMetricsConstants.CREATED_EVENT_TYPE);
    tEvent.setTimestamp(event.getTimestamp());
    entity.addEvent(tEvent);
    entity.setCreatedTime(event.getTimestamp());
    dispatcher.getEventHandler().handle(new TimelinePublishEvent(entity, containerId.getApplicationAttemptId().getApplicationId()));
}

Example 5 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class NodeStatusUpdaterImpl method getContainerStatuses.

// Iterate through the NMContext and clone and get all the containers'
// statuses. If it's a completed container, add into the
// recentlyStoppedContainers collections.
@VisibleForTesting
protected List<ContainerStatus> getContainerStatuses() throws IOException {
    List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
    for (Container container : this.context.getContainers().values()) {
        ContainerId containerId = container.getContainerId();
        ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
        org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
        if (containerStatus.getState() == ContainerState.COMPLETE) {
            if (isApplicationStopped(applicationId)) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(applicationId + " is completing, " + " remove " + containerId + " from NM context.");
                }
                context.getContainers().remove(containerId);
                pendingCompletedContainers.put(containerId, containerStatus);
            } else {
                if (!isContainerRecentlyStopped(containerId)) {
                    pendingCompletedContainers.put(containerId, containerStatus);
                }
            }
            // Adding to finished containers cache. Cache will keep it around at
            // least for #durationToTrackStoppedContainers duration. In the
            // subsequent call to stop container it will get removed from cache.
            addCompletedContainer(containerId);
        } else {
            containerStatuses.add(containerStatus);
        }
    }
    containerStatuses.addAll(pendingCompletedContainers.values());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Sending out " + containerStatuses.size() + " container statuses: " + containerStatuses);
    }
    return containerStatuses;
}