Examples with ContainerExecutionException org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException used on opensource projects

Example 6 with ContainerExecutionException

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException in project hadoop by apache.

the class TestDockerContainerRuntime method testContainerLaunchWithCustomNetworks.

@Test
@SuppressWarnings("unchecked")
public void testContainerLaunchWithCustomNetworks() throws ContainerExecutionException, IOException, PrivilegedOperationException {
    DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
    String customNetwork1 = "sdn1";
    String customNetwork2 = "sdn2";
    String customNetwork3 = "sdn3";
    String[] networks = { "host", "none", "bridge", customNetwork1, customNetwork2 };
    //customized set of allowed networks
    conf.setStrings(YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, networks);
    //default network is "sdn1"
    conf.set(YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK, customNetwork1);
    //this should cause no failures.
    runtime.initialize(conf);
    runtime.launchContainer(builder.build());
    PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
    List<String> args = op.getArguments();
    String dockerCommandFile = args.get(11);
    //This is the expected docker invocation for this case. customNetwork1
    // ("sdn1") is the expected network to be used in this case
    StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ").append("--workdir=%3$s ").append("--net=" + customNetwork1 + " ").append(getExpectedTestCapabilitiesArgumentString()).append(getExpectedCGroupsMountString()).append("-v %4$s:%4$s ").append("-v %5$s:%5$s ").append("-v %6$s:%6$s ").append("-v %7$s:%7$s ").append("-v %8$s:%8$s ").append("%9$s ").append("bash %10$s/launch_container.sh");
    String expectedCommand = String.format(expectedCommandTemplate.toString(), containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), image, containerWorkDir);
    List<String> dockerCommands = Files.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
    Assert.assertEquals(1, dockerCommands.size());
    Assert.assertEquals(expectedCommand, dockerCommands.get(0));
    //now set an explicit (non-default) allowedNetwork and ensure that it is
    // used.
    env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", customNetwork2);
    runtime.launchContainer(builder.build());
    op = capturePrivilegedOperationAndVerifyArgs();
    args = op.getArguments();
    dockerCommandFile = args.get(11);
    //This is the expected docker invocation for this case. customNetwork2
    // ("sdn2") is the expected network to be used in this case
    expectedCommandTemplate = new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ").append("--workdir=%3$s ").append("--net=" + customNetwork2 + " ").append(getExpectedTestCapabilitiesArgumentString()).append(getExpectedCGroupsMountString()).append("-v %4$s:%4$s ").append("-v %5$s:%5$s ").append("-v %6$s:%6$s ").append("-v %7$s:%7$s ").append("-v %8$s:%8$s ").append("%9$s ").append("bash %10$s/launch_container.sh");
    expectedCommand = String.format(expectedCommandTemplate.toString(), containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), image, containerWorkDir);
    dockerCommands = Files.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
    Assert.assertEquals(1, dockerCommands.size());
    Assert.assertEquals(expectedCommand, dockerCommands.get(0));
    //disallowed network should trigger a launch failure
    env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", customNetwork3);
    try {
        runtime.launchContainer(builder.build());
        Assert.fail("Disallowed network : " + customNetwork3 + "did not trigger launch failure.");
    } catch (ContainerExecutionException e) {
        LOG.info("Caught expected exception : " + e);
    }
}

Example 7 with ContainerExecutionException

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException in project hadoop by apache.

the class TestDockerContainerRuntime method testContainerLaunchWithNetworkingDefaults.

@Test
@SuppressWarnings("unchecked")
public void testContainerLaunchWithNetworkingDefaults() throws ContainerExecutionException, IOException, PrivilegedOperationException {
    DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
    runtime.initialize(conf);
    Random randEngine = new Random();
    String disallowedNetwork = "sdn" + Integer.toString(randEngine.nextInt());
    try {
        env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", disallowedNetwork);
        runtime.launchContainer(builder.build());
        Assert.fail("Network was expected to be disallowed: " + disallowedNetwork);
    } catch (ContainerExecutionException e) {
        LOG.info("Caught expected exception: " + e);
    }
    int size = YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS.length;
    String allowedNetwork = YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS[randEngine.nextInt(size)];
    env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", allowedNetwork);
    //this should cause no failures.
    runtime.launchContainer(builder.build());
    PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
    List<String> args = op.getArguments();
    String dockerCommandFile = args.get(11);
    //This is the expected docker invocation for this case
    StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ").append("--workdir=%3$s ").append("--net=" + allowedNetwork + " ").append(getExpectedTestCapabilitiesArgumentString()).append(getExpectedCGroupsMountString()).append("-v %4$s:%4$s ").append("-v %5$s:%5$s ").append("-v %6$s:%6$s ").append("-v %7$s:%7$s ").append("-v %8$s:%8$s ").append("%9$s ").append("bash %10$s/launch_container.sh");
    String expectedCommand = String.format(expectedCommandTemplate.toString(), containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), image, containerWorkDir);
    List<String> dockerCommands = Files.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
    Assert.assertEquals(1, dockerCommands.size());
    Assert.assertEquals(expectedCommand, dockerCommands.get(0));
}

Example 8 with ContainerExecutionException

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException in project hadoop by apache.

the class LinuxContainerExecutor method init.

@Override
public void init() throws IOException {
    Configuration conf = super.getConf();
    // verify configuration/permissions and exit
    try {
        PrivilegedOperation checkSetupOp = new PrivilegedOperation(PrivilegedOperation.OperationType.CHECK_SETUP);
        PrivilegedOperationExecutor privilegedOperationExecutor = PrivilegedOperationExecutor.getInstance(conf);
        privilegedOperationExecutor.executePrivilegedOperation(checkSetupOp, false);
    } catch (PrivilegedOperationException e) {
        int exitCode = e.getExitCode();
        LOG.warn("Exit code from container executor initialization is : " + exitCode, e);
        throw new IOException("Linux container executor not configured properly" + " (error=" + exitCode + ")", e);
    }
    try {
        resourceHandlerChain = ResourceHandlerModule.getConfiguredResourceHandlerChain(conf);
        if (LOG.isDebugEnabled()) {
            LOG.debug("Resource handler chain enabled = " + (resourceHandlerChain == null));
        }
        if (resourceHandlerChain != null) {
            LOG.debug("Bootstrapping resource handler chain");
            resourceHandlerChain.bootstrap(conf);
        }
    } catch (ResourceHandlerException e) {
        LOG.error("Failed to bootstrap configured resource subsystems! ", e);
        throw new IOException("Failed to bootstrap configured resource subsystems!");
    }
    try {
        if (linuxContainerRuntime == null) {
            LinuxContainerRuntime runtime = new DelegatingLinuxContainerRuntime();
            runtime.initialize(conf);
            this.linuxContainerRuntime = runtime;
        }
    } catch (ContainerExecutionException e) {
        LOG.error("Failed to initialize linux container runtime(s)!", e);
        throw new IOException("Failed to initialize linux container runtime(s)!");
    }
    resourcesHandler.init(this);
}

Example 9 with ContainerExecutionException

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException in project hadoop by apache.

the class LinuxContainerExecutor method prepareContainer.

@Override
public void prepareContainer(ContainerPrepareContext ctx) throws IOException {
    ContainerRuntimeContext.Builder builder = new ContainerRuntimeContext.Builder(ctx.getContainer());
    builder.setExecutionAttribute(LOCALIZED_RESOURCES, ctx.getLocalizedResources()).setExecutionAttribute(USER, ctx.getUser()).setExecutionAttribute(CONTAINER_LOCAL_DIRS, ctx.getContainerLocalDirs()).setExecutionAttribute(CONTAINER_RUN_CMDS, ctx.getCommands()).setExecutionAttribute(CONTAINER_ID_STR, ctx.getContainer().getContainerId().toString());
    try {
        linuxContainerRuntime.prepareContainer(builder.build());
    } catch (ContainerExecutionException e) {
        throw new IOException("Unable to prepare container: ", e);
    }
}

Example 10 with ContainerExecutionException

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException in project hadoop by apache.

the class LinuxContainerExecutor method signalContainer.

@Override
public boolean signalContainer(ContainerSignalContext ctx) throws IOException {
    Container container = ctx.getContainer();
    String user = ctx.getUser();
    String pid = ctx.getPid();
    Signal signal = ctx.getSignal();
    verifyUsernamePattern(user);
    String runAsUser = getRunAsUser(user);
    ContainerRuntimeContext runtimeContext = new ContainerRuntimeContext.Builder(container).setExecutionAttribute(RUN_AS_USER, runAsUser).setExecutionAttribute(USER, user).setExecutionAttribute(PID, pid).setExecutionAttribute(SIGNAL, signal).build();
    try {
        linuxContainerRuntime.signalContainer(runtimeContext);
    } catch (ContainerExecutionException e) {
        int retCode = e.getExitCode();
        if (retCode == PrivilegedOperation.ResultCode.INVALID_CONTAINER_PID.getValue()) {
            return false;
        }
        LOG.warn("Error in signalling container " + pid + " with " + signal + "; exit = " + retCode, e);
        logOutput(e.getOutput());
        throw new IOException("Problem signalling container " + pid + " with " + signal + "; output: " + e.getOutput() + " and exitCode: " + retCode, e);
    }
    return true;
}