Examples with Container org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container used on opensource projects

Example 6 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class NodeStatusUpdaterImpl method getContainerStatuses.

// Iterate through the NMContext and clone and get all the containers'
// statuses. If it's a completed container, add into the
// recentlyStoppedContainers collections.
@VisibleForTesting
protected List<ContainerStatus> getContainerStatuses() throws IOException {
    List<ContainerStatus> containerStatuses = new ArrayList<ContainerStatus>();
    for (Container container : this.context.getContainers().values()) {
        ContainerId containerId = container.getContainerId();
        ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
        org.apache.hadoop.yarn.api.records.ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
        if (containerStatus.getState() == ContainerState.COMPLETE) {
            if (isApplicationStopped(applicationId)) {
                if (LOG.isDebugEnabled()) {
                    LOG.debug(applicationId + " is completing, " + " remove " + containerId + " from NM context.");
                }
                context.getContainers().remove(containerId);
                pendingCompletedContainers.put(containerId, containerStatus);
            } else {
                if (!isContainerRecentlyStopped(containerId)) {
                    pendingCompletedContainers.put(containerId, containerStatus);
                }
            }
            // Adding to finished containers cache. Cache will keep it around at
            // least for #durationToTrackStoppedContainers duration. In the
            // subsequent call to stop container it will get removed from cache.
            addCompletedContainer(containerId);
        } else {
            containerStatuses.add(containerStatus);
        }
    }
    containerStatuses.addAll(pendingCompletedContainers.values());
    if (LOG.isDebugEnabled()) {
        LOG.debug("Sending out " + containerStatuses.size() + " container statuses: " + containerStatuses);
    }
    return containerStatuses;
}

Example 7 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class NodeStatusUpdaterImpl method getNMContainerStatuses.

// These NMContainerStatus are sent on NM registration and used by YARN only.
private List<NMContainerStatus> getNMContainerStatuses() throws IOException {
    List<NMContainerStatus> containerStatuses = new ArrayList<NMContainerStatus>();
    for (Container container : this.context.getContainers().values()) {
        ContainerId containerId = container.getContainerId();
        ApplicationId applicationId = containerId.getApplicationAttemptId().getApplicationId();
        if (!this.context.getApplications().containsKey(applicationId)) {
            context.getContainers().remove(containerId);
            continue;
        }
        NMContainerStatus status = container.getNMContainerStatus();
        containerStatuses.add(status);
        if (status.getContainerState() == ContainerState.COMPLETE) {
            // Adding to finished containers cache. Cache will keep it around at
            // least for #durationToTrackStoppedContainers duration. In the
            // subsequent call to stop container it will get removed from cache.
            addCompletedContainer(containerId);
        }
    }
    LOG.info("Sending out " + containerStatuses.size() + " NM container statuses: " + containerStatuses);
    return containerStatuses;
}

Example 8 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class LinuxContainerExecutor method launchContainer.

@Override
public int launchContainer(ContainerStartContext ctx) throws IOException {
    Container container = ctx.getContainer();
    Path nmPrivateContainerScriptPath = ctx.getNmPrivateContainerScriptPath();
    Path nmPrivateTokensPath = ctx.getNmPrivateTokensPath();
    String user = ctx.getUser();
    String appId = ctx.getAppId();
    Path containerWorkDir = ctx.getContainerWorkDir();
    List<String> localDirs = ctx.getLocalDirs();
    List<String> logDirs = ctx.getLogDirs();
    List<String> filecacheDirs = ctx.getFilecacheDirs();
    List<String> userLocalDirs = ctx.getUserLocalDirs();
    List<String> containerLocalDirs = ctx.getContainerLocalDirs();
    List<String> containerLogDirs = ctx.getContainerLogDirs();
    Map<Path, List<String>> localizedResources = ctx.getLocalizedResources();
    verifyUsernamePattern(user);
    String runAsUser = getRunAsUser(user);
    ContainerId containerId = container.getContainerId();
    String containerIdStr = containerId.toString();
    resourcesHandler.preExecute(containerId, container.getResource());
    String resourcesOptions = resourcesHandler.getResourcesOption(containerId);
    String tcCommandFile = null;
    try {
        if (resourceHandlerChain != null) {
            List<PrivilegedOperation> ops = resourceHandlerChain.preStart(container);
            if (ops != null) {
                List<PrivilegedOperation> resourceOps = new ArrayList<>();
                resourceOps.add(new PrivilegedOperation(PrivilegedOperation.OperationType.ADD_PID_TO_CGROUP, resourcesOptions));
                for (PrivilegedOperation op : ops) {
                    switch(op.getOperationType()) {
                        case ADD_PID_TO_CGROUP:
                            resourceOps.add(op);
                            break;
                        case TC_MODIFY_STATE:
                            tcCommandFile = op.getArguments().get(0);
                            break;
                        default:
                            LOG.warn("PrivilegedOperation type unsupported in launch: " + op.getOperationType());
                    }
                }
                if (resourceOps.size() > 1) {
                    //squash resource operations
                    try {
                        PrivilegedOperation operation = PrivilegedOperationExecutor.squashCGroupOperations(resourceOps);
                        resourcesOptions = operation.getArguments().get(0);
                    } catch (PrivilegedOperationException e) {
                        LOG.error("Failed to squash cgroup operations!", e);
                        throw new ResourceHandlerException("Failed to squash cgroup operations!");
                    }
                }
            }
        }
    } catch (ResourceHandlerException e) {
        LOG.error("ResourceHandlerChain.preStart() failed!", e);
        throw new IOException("ResourceHandlerChain.preStart() failed!", e);
    }
    try {
        Path pidFilePath = getPidFilePath(containerId);
        if (pidFilePath != null) {
            List<String> prefixCommands = new ArrayList<>();
            ContainerRuntimeContext.Builder builder = new ContainerRuntimeContext.Builder(container);
            addSchedPriorityCommand(prefixCommands);
            if (prefixCommands.size() > 0) {
                builder.setExecutionAttribute(CONTAINER_LAUNCH_PREFIX_COMMANDS, prefixCommands);
            }
            builder.setExecutionAttribute(LOCALIZED_RESOURCES, localizedResources).setExecutionAttribute(RUN_AS_USER, runAsUser).setExecutionAttribute(USER, user).setExecutionAttribute(APPID, appId).setExecutionAttribute(CONTAINER_ID_STR, containerIdStr).setExecutionAttribute(CONTAINER_WORK_DIR, containerWorkDir).setExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH, nmPrivateContainerScriptPath).setExecutionAttribute(NM_PRIVATE_TOKENS_PATH, nmPrivateTokensPath).setExecutionAttribute(PID_FILE_PATH, pidFilePath).setExecutionAttribute(LOCAL_DIRS, localDirs).setExecutionAttribute(LOG_DIRS, logDirs).setExecutionAttribute(FILECACHE_DIRS, filecacheDirs).setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs).setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs).setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs).setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
            if (tcCommandFile != null) {
                builder.setExecutionAttribute(TC_COMMAND_FILE, tcCommandFile);
            }
            linuxContainerRuntime.launchContainer(builder.build());
        } else {
            LOG.info("Container was marked as inactive. Returning terminated error");
            return ExitCode.TERMINATED.getExitCode();
        }
    } catch (ContainerExecutionException e) {
        int exitCode = e.getExitCode();
        LOG.warn("Exit code from container " + containerId + " is : " + exitCode);
        // output
        if (exitCode != ExitCode.FORCE_KILLED.getExitCode() && exitCode != ExitCode.TERMINATED.getExitCode()) {
            LOG.warn("Exception from container-launch with container ID: " + containerId + " and exit code: " + exitCode, e);
            StringBuilder builder = new StringBuilder();
            builder.append("Exception from container-launch.\n");
            builder.append("Container id: " + containerId + "\n");
            builder.append("Exit code: " + exitCode + "\n");
            if (!Optional.fromNullable(e.getErrorOutput()).or("").isEmpty()) {
                builder.append("Exception message: " + e.getErrorOutput() + "\n");
            }
            builder.append("Stack trace: " + StringUtils.stringifyException(e) + "\n");
            if (!e.getOutput().isEmpty()) {
                builder.append("Shell output: " + e.getOutput() + "\n");
            }
            String diagnostics = builder.toString();
            logOutput(diagnostics);
            container.handle(new ContainerDiagnosticsUpdateEvent(containerId, diagnostics));
        } else {
            container.handle(new ContainerDiagnosticsUpdateEvent(containerId, "Container killed on request. Exit code is " + exitCode));
        }
        return exitCode;
    } finally {
        resourcesHandler.postExecute(containerId);
        try {
            if (resourceHandlerChain != null) {
                resourceHandlerChain.postComplete(containerId);
            }
        } catch (ResourceHandlerException e) {
            LOG.warn("ResourceHandlerChain.postComplete failed for " + "containerId: " + containerId + ". Exception: " + e);
        }
    }
    return 0;
}

Example 9 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestContainerLogsPage method testContainerLogPageAccess.

@Test(timeout = 10000)
public void testContainerLogPageAccess() throws IOException {
    // SecureIOUtils require Native IO to be enabled. This test will run
    // only if it is enabled.
    assumeTrue(NativeIO.isAvailable());
    String user = "randomUser" + System.currentTimeMillis();
    File absLogDir = null, appDir = null, containerDir = null, syslog = null;
    try {
        // target log directory
        absLogDir = new File("target", TestContainerLogsPage.class.getSimpleName() + "LogDir").getAbsoluteFile();
        absLogDir.mkdir();
        Configuration conf = new Configuration();
        conf.set(YarnConfiguration.NM_LOG_DIRS, absLogDir.toURI().toString());
        conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
        UserGroupInformation.setConfiguration(conf);
        NodeHealthCheckerService healthChecker = createNodeHealthCheckerService(conf);
        healthChecker.init(conf);
        LocalDirsHandlerService dirsHandler = healthChecker.getDiskHandler();
        // Add an application and the corresponding containers
        RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(conf);
        long clusterTimeStamp = 1234;
        ApplicationId appId = BuilderUtils.newApplicationId(recordFactory, clusterTimeStamp, 1);
        Application app = mock(Application.class);
        when(app.getAppId()).thenReturn(appId);
        // Making sure that application returns a random user. This is required
        // for SecureIOUtils' file owner check.
        when(app.getUser()).thenReturn(user);
        ApplicationAttemptId appAttemptId = BuilderUtils.newApplicationAttemptId(appId, 1);
        ContainerId container1 = BuilderUtils.newContainerId(recordFactory, appId, appAttemptId, 0);
        // Testing secure read access for log files
        // Creating application and container directory and syslog file.
        appDir = new File(absLogDir, appId.toString());
        appDir.mkdir();
        containerDir = new File(appDir, container1.toString());
        containerDir.mkdir();
        syslog = new File(containerDir, "syslog");
        syslog.createNewFile();
        BufferedOutputStream out = new BufferedOutputStream(new FileOutputStream(syslog));
        out.write("Log file Content".getBytes());
        out.close();
        Context context = mock(Context.class);
        ConcurrentMap<ApplicationId, Application> appMap = new ConcurrentHashMap<ApplicationId, Application>();
        appMap.put(appId, app);
        when(context.getApplications()).thenReturn(appMap);
        ConcurrentHashMap<ContainerId, Container> containers = new ConcurrentHashMap<ContainerId, Container>();
        when(context.getContainers()).thenReturn(containers);
        when(context.getLocalDirsHandler()).thenReturn(dirsHandler);
        MockContainer container = new MockContainer(appAttemptId, new AsyncDispatcher(), conf, user, appId, 1);
        container.setState(ContainerState.RUNNING);
        context.getContainers().put(container1, container);
        ContainersLogsBlock cLogsBlock = new ContainersLogsBlock(context);
        Map<String, String> params = new HashMap<String, String>();
        params.put(YarnWebParams.CONTAINER_ID, container1.toString());
        params.put(YarnWebParams.CONTAINER_LOG_TYPE, "syslog");
        Injector injector = WebAppTests.testPage(ContainerLogsPage.class, ContainersLogsBlock.class, cLogsBlock, params, (Module[]) null);
        PrintWriter spyPw = WebAppTests.getPrintWriter(injector);
        verify(spyPw).write("Exception reading log file. Application submitted by '" + user + "' doesn't own requested log file : syslog");
    } finally {
        if (syslog != null) {
            syslog.delete();
        }
        if (containerDir != null) {
            containerDir.delete();
        }
        if (appDir != null) {
            appDir.delete();
        }
        if (absLogDir != null) {
            absLogDir.delete();
        }
    }
}

Example 10 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestContainersMonitorResourceChange method setup.

@Before
public void setup() {
    executor = new MockExecutor();
    dispatcher = new AsyncDispatcher();
    context = Mockito.mock(Context.class);
    containerMap = new ConcurrentSkipListMap<>();
    Container container = Mockito.mock(ContainerImpl.class);
    containerMap.put(getContainerId(1), container);
    Mockito.doReturn(containerMap).when(context).getContainers();
    conf = new Configuration();
    conf.set(YarnConfiguration.NM_CONTAINER_MON_RESOURCE_CALCULATOR, MockResourceCalculatorPlugin.class.getCanonicalName());
    conf.set(YarnConfiguration.NM_CONTAINER_MON_PROCESS_TREE, MockResourceCalculatorProcessTree.class.getCanonicalName());
    dispatcher.init(conf);
    dispatcher.start();
    containerEventHandler = new MockContainerEventHandler();
    dispatcher.register(ContainerEventType.class, containerEventHandler);
}