Examples with Container org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container used on opensource projects

Example 21 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class ContainerManagerImpl method localize.

@Override
@SuppressWarnings("unchecked")
public ResourceLocalizationResponse localize(ResourceLocalizationRequest request) throws YarnException, IOException {
    ContainerId containerId = request.getContainerId();
    Container container = preReInitializeOrLocalizeCheck(containerId, ReInitOp.LOCALIZE);
    try {
        Map<LocalResourceVisibility, Collection<LocalResourceRequest>> req = container.getResourceSet().addResources(request.getLocalResources());
        if (req != null && !req.isEmpty()) {
            dispatcher.getEventHandler().handle(new ContainerLocalizationRequestEvent(container, req));
        }
    } catch (URISyntaxException e) {
        LOG.info("Error when parsing local resource URI for " + containerId, e);
        throw new YarnException(e);
    }
    return ResourceLocalizationResponse.newInstance();
}

Example 22 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class LogAggregationService method stopContainer.

private void stopContainer(ContainerId containerId, int exitCode) {
    // A container is complete. Put this containers' logs up for aggregation if
    // this containers' logs are needed.
    AppLogAggregator aggregator = this.appLogAggregators.get(containerId.getApplicationAttemptId().getApplicationId());
    if (aggregator == null) {
        LOG.warn("Log aggregation is not initialized for " + containerId + ", did it fail to start?");
        return;
    }
    Container container = context.getContainers().get(containerId);
    if (null == container) {
        LOG.warn("Log aggregation cannot be started for " + containerId + ", as its an absent container");
        return;
    }
    ContainerType containerType = container.getContainerTokenIdentifier().getContainerType();
    aggregator.startContainerLogAggregation(new ContainerLogContext(containerId, containerType, exitCode));
}

Example 23 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class DefaultLinuxContainerRuntime method launchContainer.

@Override
public void launchContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException {
    Container container = ctx.getContainer();
    PrivilegedOperation launchOp = new PrivilegedOperation(PrivilegedOperation.OperationType.LAUNCH_CONTAINER);
    //All of these arguments are expected to be available in the runtime context
    launchOp.appendArgs(ctx.getExecutionAttribute(RUN_AS_USER), ctx.getExecutionAttribute(USER), Integer.toString(PrivilegedOperation.RunAsUserCommand.LAUNCH_CONTAINER.getValue()), ctx.getExecutionAttribute(APPID), ctx.getExecutionAttribute(CONTAINER_ID_STR), ctx.getExecutionAttribute(CONTAINER_WORK_DIR).toString(), ctx.getExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH).toUri().getPath(), ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(), ctx.getExecutionAttribute(PID_FILE_PATH).toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, ctx.getExecutionAttribute(LOCAL_DIRS)), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, ctx.getExecutionAttribute(LOG_DIRS)), ctx.getExecutionAttribute(RESOURCES_OPTIONS));
    String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);
    if (tcCommandFile != null) {
        launchOp.appendArgs(tcCommandFile);
    }
    //List<String> -> stored as List -> fetched/converted to List<String>
    //we can't do better here thanks to type-erasure
    @SuppressWarnings("unchecked") List<String> prefixCommands = (List<String>) ctx.getExecutionAttribute(CONTAINER_LAUNCH_PREFIX_COMMANDS);
    try {
        privilegedOperationExecutor.executePrivilegedOperation(prefixCommands, launchOp, null, container.getLaunchContext().getEnvironment(), false, false);
    } catch (PrivilegedOperationException e) {
        LOG.warn("Launch container failed. Exception: ", e);
        throw new ContainerExecutionException("Launch container failed", e.getExitCode(), e.getOutput(), e.getErrorOutput());
    }
}

Example 24 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class DockerLinuxContainerRuntime method launchContainer.

@Override
public void launchContainer(ContainerRuntimeContext ctx) throws ContainerExecutionException {
    Container container = ctx.getContainer();
    Map<String, String> environment = container.getLaunchContext().getEnvironment();
    String imageName = environment.get(ENV_DOCKER_CONTAINER_IMAGE);
    String network = environment.get(ENV_DOCKER_CONTAINER_NETWORK);
    if (network == null || network.isEmpty()) {
        network = defaultNetwork;
    }
    validateContainerNetworkType(network);
    if (imageName == null) {
        throw new ContainerExecutionException(ENV_DOCKER_CONTAINER_IMAGE + " not set!");
    }
    String containerIdStr = container.getContainerId().toString();
    String runAsUser = ctx.getExecutionAttribute(RUN_AS_USER);
    Path containerWorkDir = ctx.getExecutionAttribute(CONTAINER_WORK_DIR);
    //List<String> -> stored as List -> fetched/converted to List<String>
    //we can't do better here thanks to type-erasure
    @SuppressWarnings("unchecked") List<String> localDirs = ctx.getExecutionAttribute(LOCAL_DIRS);
    @SuppressWarnings("unchecked") List<String> logDirs = ctx.getExecutionAttribute(LOG_DIRS);
    @SuppressWarnings("unchecked") List<String> filecacheDirs = ctx.getExecutionAttribute(FILECACHE_DIRS);
    @SuppressWarnings("unchecked") List<String> containerLocalDirs = ctx.getExecutionAttribute(CONTAINER_LOCAL_DIRS);
    @SuppressWarnings("unchecked") List<String> containerLogDirs = ctx.getExecutionAttribute(CONTAINER_LOG_DIRS);
    @SuppressWarnings("unchecked") Map<Path, List<String>> localizedResources = ctx.getExecutionAttribute(LOCALIZED_RESOURCES);
    @SuppressWarnings("unchecked") List<String> userLocalDirs = ctx.getExecutionAttribute(USER_LOCAL_DIRS);
    Set<String> capabilities = new HashSet<>(Arrays.asList(conf.getTrimmedStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, YarnConfiguration.DEFAULT_NM_DOCKER_CONTAINER_CAPABILITIES)));
    @SuppressWarnings("unchecked") DockerRunCommand runCommand = new DockerRunCommand(containerIdStr, runAsUser, imageName).detachOnRun().setContainerWorkDir(containerWorkDir.toString()).setNetworkType(network).setCapabilities(capabilities).addMountLocation(CGROUPS_ROOT_DIRECTORY, CGROUPS_ROOT_DIRECTORY + ":ro", false);
    List<String> allDirs = new ArrayList<>(containerLocalDirs);
    allDirs.addAll(filecacheDirs);
    allDirs.add(containerWorkDir.toString());
    allDirs.addAll(containerLogDirs);
    allDirs.addAll(userLocalDirs);
    for (String dir : allDirs) {
        runCommand.addMountLocation(dir, dir, true);
    }
    if (environment.containsKey(ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS)) {
        String mounts = environment.get(ENV_DOCKER_CONTAINER_LOCAL_RESOURCE_MOUNTS);
        if (!mounts.isEmpty()) {
            for (String mount : StringUtils.split(mounts)) {
                String[] dir = StringUtils.split(mount, ':');
                if (dir.length != 2) {
                    throw new ContainerExecutionException("Invalid mount : " + mount);
                }
                String src = validateMount(dir[0], localizedResources);
                String dst = dir[1];
                runCommand.addMountLocation(src, dst + ":ro", true);
            }
        }
    }
    if (allowPrivilegedContainerExecution(container)) {
        runCommand.setPrivileged();
    }
    String resourcesOpts = ctx.getExecutionAttribute(RESOURCES_OPTIONS);
    addCGroupParentIfRequired(resourcesOpts, containerIdStr, runCommand);
    Path nmPrivateContainerScriptPath = ctx.getExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH);
    String disableOverride = environment.get(ENV_DOCKER_CONTAINER_RUN_OVERRIDE_DISABLE);
    if (disableOverride != null && disableOverride.equals("true")) {
        if (LOG.isInfoEnabled()) {
            LOG.info("command override disabled");
        }
    } else {
        List<String> overrideCommands = new ArrayList<>();
        Path launchDst = new Path(containerWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
        overrideCommands.add("bash");
        overrideCommands.add(launchDst.toUri().getPath());
        runCommand.setOverrideCommandWithArgs(overrideCommands);
    }
    String commandFile = dockerClient.writeCommandToTempFile(runCommand, containerIdStr);
    PrivilegedOperation launchOp = new PrivilegedOperation(PrivilegedOperation.OperationType.LAUNCH_DOCKER_CONTAINER);
    launchOp.appendArgs(runAsUser, ctx.getExecutionAttribute(USER), Integer.toString(PrivilegedOperation.RunAsUserCommand.LAUNCH_DOCKER_CONTAINER.getValue()), ctx.getExecutionAttribute(APPID), containerIdStr, containerWorkDir.toString(), nmPrivateContainerScriptPath.toUri().getPath(), ctx.getExecutionAttribute(NM_PRIVATE_TOKENS_PATH).toUri().getPath(), ctx.getExecutionAttribute(PID_FILE_PATH).toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, localDirs), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, logDirs), commandFile, resourcesOpts);
    String tcCommandFile = ctx.getExecutionAttribute(TC_COMMAND_FILE);
    if (tcCommandFile != null) {
        launchOp.appendArgs(tcCommandFile);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Launching container with cmd: " + runCommand.getCommandWithArguments());
    }
    try {
        privilegedOperationExecutor.executePrivilegedOperation(null, launchOp, null, container.getLaunchContext().getEnvironment(), false, false);
    } catch (PrivilegedOperationException e) {
        LOG.warn("Launch container failed. Exception: ", e);
        LOG.info("Docker command used: " + runCommand.getCommandWithArguments());
        throw new ContainerExecutionException("Launch container failed", e.getExitCode(), e.getOutput(), e.getErrorOutput());
    }
}

Example 25 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class ContainerManagerImpl method waitForRecoveredContainers.

private void waitForRecoveredContainers() throws InterruptedException {
    final int sleepMsec = 100;
    int waitIterations = 100;
    List<ContainerId> newContainers = new ArrayList<ContainerId>();
    while (--waitIterations >= 0) {
        newContainers.clear();
        for (Container container : context.getContainers().values()) {
            if (container.getContainerState() == org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState.NEW) {
                newContainers.add(container.getContainerId());
            }
        }
        if (newContainers.isEmpty()) {
            break;
        }
        LOG.info("Waiting for containers: " + newContainers);
        Thread.sleep(sleepMsec);
    }
    if (waitIterations < 0) {
        LOG.warn("Timeout waiting for recovered containers");
    }
}