Examples with ContainerExecutionException org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException used on opensource projects

Example 1 with ContainerExecutionException

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException in project hadoop by apache.

the class LinuxContainerExecutor method launchContainer.

@Override
public int launchContainer(ContainerStartContext ctx) throws IOException {
    Container container = ctx.getContainer();
    Path nmPrivateContainerScriptPath = ctx.getNmPrivateContainerScriptPath();
    Path nmPrivateTokensPath = ctx.getNmPrivateTokensPath();
    String user = ctx.getUser();
    String appId = ctx.getAppId();
    Path containerWorkDir = ctx.getContainerWorkDir();
    List<String> localDirs = ctx.getLocalDirs();
    List<String> logDirs = ctx.getLogDirs();
    List<String> filecacheDirs = ctx.getFilecacheDirs();
    List<String> userLocalDirs = ctx.getUserLocalDirs();
    List<String> containerLocalDirs = ctx.getContainerLocalDirs();
    List<String> containerLogDirs = ctx.getContainerLogDirs();
    Map<Path, List<String>> localizedResources = ctx.getLocalizedResources();
    verifyUsernamePattern(user);
    String runAsUser = getRunAsUser(user);
    ContainerId containerId = container.getContainerId();
    String containerIdStr = containerId.toString();
    resourcesHandler.preExecute(containerId, container.getResource());
    String resourcesOptions = resourcesHandler.getResourcesOption(containerId);
    String tcCommandFile = null;
    try {
        if (resourceHandlerChain != null) {
            List<PrivilegedOperation> ops = resourceHandlerChain.preStart(container);
            if (ops != null) {
                List<PrivilegedOperation> resourceOps = new ArrayList<>();
                resourceOps.add(new PrivilegedOperation(PrivilegedOperation.OperationType.ADD_PID_TO_CGROUP, resourcesOptions));
                for (PrivilegedOperation op : ops) {
                    switch(op.getOperationType()) {
                        case ADD_PID_TO_CGROUP:
                            resourceOps.add(op);
                            break;
                        case TC_MODIFY_STATE:
                            tcCommandFile = op.getArguments().get(0);
                            break;
                        default:
                            LOG.warn("PrivilegedOperation type unsupported in launch: " + op.getOperationType());
                    }
                }
                if (resourceOps.size() > 1) {
                    //squash resource operations
                    try {
                        PrivilegedOperation operation = PrivilegedOperationExecutor.squashCGroupOperations(resourceOps);
                        resourcesOptions = operation.getArguments().get(0);
                    } catch (PrivilegedOperationException e) {
                        LOG.error("Failed to squash cgroup operations!", e);
                        throw new ResourceHandlerException("Failed to squash cgroup operations!");
                    }
                }
            }
        }
    } catch (ResourceHandlerException e) {
        LOG.error("ResourceHandlerChain.preStart() failed!", e);
        throw new IOException("ResourceHandlerChain.preStart() failed!", e);
    }
    try {
        Path pidFilePath = getPidFilePath(containerId);
        if (pidFilePath != null) {
            List<String> prefixCommands = new ArrayList<>();
            ContainerRuntimeContext.Builder builder = new ContainerRuntimeContext.Builder(container);
            addSchedPriorityCommand(prefixCommands);
            if (prefixCommands.size() > 0) {
                builder.setExecutionAttribute(CONTAINER_LAUNCH_PREFIX_COMMANDS, prefixCommands);
            }
            builder.setExecutionAttribute(LOCALIZED_RESOURCES, localizedResources).setExecutionAttribute(RUN_AS_USER, runAsUser).setExecutionAttribute(USER, user).setExecutionAttribute(APPID, appId).setExecutionAttribute(CONTAINER_ID_STR, containerIdStr).setExecutionAttribute(CONTAINER_WORK_DIR, containerWorkDir).setExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH, nmPrivateContainerScriptPath).setExecutionAttribute(NM_PRIVATE_TOKENS_PATH, nmPrivateTokensPath).setExecutionAttribute(PID_FILE_PATH, pidFilePath).setExecutionAttribute(LOCAL_DIRS, localDirs).setExecutionAttribute(LOG_DIRS, logDirs).setExecutionAttribute(FILECACHE_DIRS, filecacheDirs).setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs).setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs).setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs).setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
            if (tcCommandFile != null) {
                builder.setExecutionAttribute(TC_COMMAND_FILE, tcCommandFile);
            }
            linuxContainerRuntime.launchContainer(builder.build());
        } else {
            LOG.info("Container was marked as inactive. Returning terminated error");
            return ExitCode.TERMINATED.getExitCode();
        }
    } catch (ContainerExecutionException e) {
        int exitCode = e.getExitCode();
        LOG.warn("Exit code from container " + containerId + " is : " + exitCode);
        // output
        if (exitCode != ExitCode.FORCE_KILLED.getExitCode() && exitCode != ExitCode.TERMINATED.getExitCode()) {
            LOG.warn("Exception from container-launch with container ID: " + containerId + " and exit code: " + exitCode, e);
            StringBuilder builder = new StringBuilder();
            builder.append("Exception from container-launch.\n");
            builder.append("Container id: " + containerId + "\n");
            builder.append("Exit code: " + exitCode + "\n");
            if (!Optional.fromNullable(e.getErrorOutput()).or("").isEmpty()) {
                builder.append("Exception message: " + e.getErrorOutput() + "\n");
            }
            builder.append("Stack trace: " + StringUtils.stringifyException(e) + "\n");
            if (!e.getOutput().isEmpty()) {
                builder.append("Shell output: " + e.getOutput() + "\n");
            }
            String diagnostics = builder.toString();
            logOutput(diagnostics);
            container.handle(new ContainerDiagnosticsUpdateEvent(containerId, diagnostics));
        } else {
            container.handle(new ContainerDiagnosticsUpdateEvent(containerId, "Container killed on request. Exit code is " + exitCode));
        }
        return exitCode;
    } finally {
        resourcesHandler.postExecute(containerId);
        try {
            if (resourceHandlerChain != null) {
                resourceHandlerChain.postComplete(containerId);
            }
        } catch (ResourceHandlerException e) {
            LOG.warn("ResourceHandlerChain.postComplete failed for " + "containerId: " + containerId + ". Exception: " + e);
        }
    }
    return 0;
}

Example 2 with ContainerExecutionException

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException in project hadoop by apache.

the class DefaultLinuxContainerRuntime method launchContainer.

@Override
public void launchContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException {
    Container container = ctx.getContainer();
    PrivilegedOperation launchOp = new PrivilegedOperation(PrivilegedOperation.OperationType.LAUNCH_CONTAINER);
    //All of these arguments are expected to be available in the runtime context
    launchOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER), ctx.getExecutionAttribute(USER), Integer.toString(PrivilegedOperation.RunAsUserCommand.LAUNCH_CONTAINER.getValue()), ctx.getExecutionAttribute(APPID), ctx.getExecutionAttribute(CONTAINER_ID_STR), ctx.getExecutionAttribute(CONTAINER_WORK_DIR).toString(), ctx.getExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH).toUri().getPath(), ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(), ctx.getExecutionAttribute(PID_FILE_PATH).toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, ctx.getExecutionAttribute(LOCAL_DIRS)), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, ctx.getExecutionAttribute(LOG_DIRS)), ctx.getExecutionAttribute(RESOURCES_OPTIONS));
    String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);
    if (tcCommandFile != null) {
        launchOp.appendArgs(tcCommandFile);
    }
    //List<String> -> stored as List -> fetched/converted to List<String>
    //we can't do better here thanks to type-erasure
    @SuppressWarnings("unchecked") List<String> prefixCommands = (List<String>) ctx.getExecutionAttribute(CONTAINER_LAUNCH_PREFIX_COMMANDS);
    try {
        privilegedOperationExecutor.executePrivilegedOperation(prefixCommands, launchOp, null, container.getLaunchContext().getEnvironment(), false, false);
    } catch (PrivilegedOperationException e) {
        LOG.warn("Launch container failed. Exception: ", e);
        throw new ContainerExecutionException("Launch container failed", e.getExitCode(), e.getOutput(), e.getErrorOutput());
    }
}

Example 3 with ContainerExecutionException

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException in project hadoop by apache.

the class DockerLinuxContainerRuntime method launchContainer.

@Override
public void launchContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException {
    Container container = ctx.getContainer();
    Map<String, String> environment = container.getLaunchContext().getEnvironment();
    String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
    String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK);
    if (network == null || network.isEmpty()) {
        network = defaultNetwork;
    }
    validateContainerNetworkType(network);
    if (imageName == null) {
        throw new ContainerExecutionException(ENV_DOCKER_CONTAINER_IMAGE + " not set!");
    }
    String containerIdStr = container.getContainerId().toString();
    String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
    Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
    //List<String> -> stored as List -> fetched/converted to List<String>
    //we can't do better here thanks to type-erasure
    @SuppressWarnings("unchecked") List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
    @SuppressWarnings("unchecked") List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
    @SuppressWarnings("unchecked") List<String> filecacheDirs = ctx.getExecutionAttribute(FILECACHE_DIRS);
    @SuppressWarnings("unchecked") List<String> containerLocalDirs = ctx.getExecutionAttribute(CONTAINER_LOCAL_DIRS);
    @SuppressWarnings("unchecked") List<String> containerLogDirs = ctx.getExecutionAttribute(CONTAINER_LOG_DIRS);
    @SuppressWarnings("unchecked") Map<Path, List<String>> localizedResources = ctx.getExecutionAttribute(LOCALIZED_RESOURCES);
    @SuppressWarnings("unchecked") List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
    Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getTrimmedStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
    @SuppressWarnings("unchecked") DockerRunCommand runCommand = new DockerRunCommand(containerIdStr, runAsUser, imageName).detachOnRun().setContainerWorkDir(containerWorkDir.toString()).setNetworkType(network).setCapabilities(capabilities).addMountLocation(CGROUPS_ROOT_DIRECTORY, CGROUPS_ROOT_DIRECTORY + ":ro", false);
    List<String> allDirs = new ArrayList<>(containerLocalDirs);
    allDirs.addAll(filecacheDirs);
    allDirs.add(containerWorkDir.toString());
    allDirs.addAll(containerLogDirs);
    allDirs.addAll(userLocalDirs);
    for (String dir : allDirs) {
        runCommand.addMountLocation(dir, dir, true);
    }
    if (environment.containsKey(ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS)) {
        String mounts = environment.get(ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS);
        if (!mounts.isEmpty()) {
            for (String mount : StringUtils.split(mounts)) {
                String[] dir = StringUtils.split(mount, ':');
                if (dir.length != 2) {
                    throw new ContainerExecutionException("Invalid mount : " + mount);
                }
                String src = validateMount(dir[0], localizedResources);
                String dst = dir[1];
                runCommand.addMountLocation(src, dst + ":ro", true);
            }
        }
    }
    if (allowPrivilegedContainerExecution(container)) {
        runCommand.setPrivileged();
    }
    String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS);
    addCGroupParentIfRequired(resourcesOpts, containerIdStr, runCommand);
    Path nmPrivateContainerScriptPath = ctx.getExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH);
    String disableOverride = environment.get(ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE);
    if (disableOverride != null && disableOverride.equals("true")) {
        if (LOG.isInfoEnabled()) {
            LOG.info("command override disabled");
        }
    } else {
        List<String> overrideCommands = new ArrayList<>();
        Path launchDst = new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
        overrideCommands.add("bash");
        overrideCommands.add(launchDst.toUri().getPath());
        runCommand.setOverrideCommandWithArgs(overrideCommands);
    }
    String commandFile = dockerClient.writeCommandToTempFile(runCommand, containerIdStr);
    PrivilegedOperation launchOp = new PrivilegedOperation(PrivilegedOperation.OperationType.LAUNCH_DOCKER_CONTAINER);
    launchOp.appendArgs(runAsUser, ctx.getExecutionAttribute(USER), Integer.toString(PrivilegedOperation.RunAsUserCommand.LAUNCH_DOCKER_CONTAINER.getValue()), ctx.getExecutionAttribute(APPID), containerIdStr, containerWorkDir.toString(), nmPrivateContainerScriptPath.toUri().getPath(), ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(), ctx.getExecutionAttribute(PID_FILE_PATH).toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, localDirs), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, logDirs), commandFile, resourcesOpts);
    String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);
    if (tcCommandFile != null) {
        launchOp.appendArgs(tcCommandFile);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Launching container with cmd: " + runCommand.getCommandWithArguments());
    }
    try {
        privilegedOperationExecutor.executePrivilegedOperation(null, launchOp, null, container.getLaunchContext().getEnvironment(), false, false);
    } catch (PrivilegedOperationException e) {
        LOG.warn("Launch container failed. Exception: ", e);
        LOG.info("Docker command used: " + runCommand.getCommandWithArguments());
        throw new ContainerExecutionException("Launch container failed", e.getExitCode(), e.getOutput(), e.getErrorOutput());
    }
}

Example 4 with ContainerExecutionException

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException in project hadoop by apache.

the class DockerLinuxContainerRuntime method allowPrivilegedContainerExecution.

/**
   * Return whether the YARN container is allowed to run in a privileged
   * Docker container. For a privileged container to be allowed all of the
   * following three conditions must be satisfied:
   *
   * <ol>
   *   <li>Submitting user must request for a privileged container</li>
   *   <li>Privileged containers must be enabled on the cluster</li>
   *   <li>Submitting user must be white-listed to run a privileged
   *   container</li>
   * </ol>
   *
   * @param container the target YARN container
   * @return whether privileged container execution is allowed
   * @throws ContainerExecutionException if privileged container execution
   * is requested but is not allowed
   */
private boolean allowPrivilegedContainerExecution(Container container) throws ContainerExecutionException {
    Map<String, String> environment = container.getLaunchContext().getEnvironment();
    String runPrivilegedContainerEnvVar = environment.get(ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER);
    if (runPrivilegedContainerEnvVar == null) {
        return false;
    }
    if (!runPrivilegedContainerEnvVar.equalsIgnoreCase("true")) {
        LOG.warn("NOT running a privileged container. Value of " + ENV_DOCKER_CONTAINER_RUN_PRIVILEGED_CONTAINER + "is invalid: " + runPrivilegedContainerEnvVar);
        return false;
    }
    if (LOG.isInfoEnabled()) {
        LOG.info("Privileged container requested for : " + container.getContainerId().toString());
    }
    //Ok, so we have been asked to run a privileged container. Security
    // checks need to be run. Each violation is an error.
    //check if privileged containers are enabled.
    boolean privilegedContainersEnabledOnCluster = conf.getBoolean(YarnConfiguration.NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS, YarnConfiguration.DEFAULT_NM_DOCKER_ALLOW_PRIVILEGED_CONTAINERS);
    if (!privilegedContainersEnabledOnCluster) {
        String message = "Privileged container being requested but privileged " + "containers are not enabled on this cluster";
        LOG.warn(message);
        throw new ContainerExecutionException(message);
    }
    //check if submitting user is in the whitelist.
    String submittingUser = container.getUser();
    UserGroupInformation submitterUgi = UserGroupInformation.createRemoteUser(submittingUser);
    if (!privilegedContainersAcl.isUserAllowed(submitterUgi)) {
        String message = "Cannot launch privileged container. Submitting user (" + submittingUser + ") fails ACL check.";
        LOG.warn(message);
        throw new ContainerExecutionException(message);
    }
    if (LOG.isInfoEnabled()) {
        LOG.info("All checks pass. Launching privileged container for : " + container.getContainerId().toString());
    }
    return true;
}

Example 5 with ContainerExecutionException

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException in project hadoop by apache.

the class JavaSandboxLinuxContainerRuntime method isSandboxContainerWhitelisted.

/**
   * Determine if the container should be whitelisted (i.e. exempt from the
   * Java Security Manager).
   * @param ctx The container runtime context for the requested container
   * @param commands The list of run commands for the container
   * @return boolean value denoting whether the container should be whitelisted.
   * @throws ContainerExecutionException If container user can not be resolved
   */
private boolean isSandboxContainerWhitelisted(ContainerRuntimeContext ctx, List<String> commands) throws ContainerExecutionException {
    String whitelistGroup = configuration.get(YarnConfiguration.YARN_CONTAINER_SANDBOX_WHITELIST_GROUP);
    Groups groups = Groups.getUserToGroupsMappingService(configuration);
    List<String> userGroups;
    boolean isWhitelisted = false;
    try {
        userGroups = groups.getGroups(ctx.getExecutionAttribute(USER));
    } catch (IOException e) {
        throw new ContainerExecutionException("Container user does not exist");
    }
    if (whitelistGroup != null && userGroups.contains(whitelistGroup)) {
        // If any command has security flag, whitelisting is disabled
        for (String cmd : commands) {
            if (cmd.contains(NMContainerPolicyUtils.SECURITY_FLAG)) {
                isWhitelisted = false;
                break;
            } else {
                isWhitelisted = true;
            }
        }
    }
    return isWhitelisted;
}