Examples with PrivilegedOperation org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation used on opensource projects

Example 6 with PrivilegedOperation

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation in project hadoop by apache.

the class CGroupsMemoryResourceHandlerImpl method preStart.

@Override
public List<PrivilegedOperation> preStart(Container container) throws ResourceHandlerException {
    String cgroupId = container.getContainerId().toString();
    //memory is in MB
    long containerSoftLimit = (long) (container.getResource().getMemorySize() * this.softLimit);
    long containerHardLimit = container.getResource().getMemorySize();
    cGroupsHandler.createCGroup(MEMORY, cgroupId);
    try {
        cGroupsHandler.updateCGroupParam(MEMORY, cgroupId, CGroupsHandler.CGROUP_PARAM_MEMORY_HARD_LIMIT_BYTES, String.valueOf(containerHardLimit) + "M");
        cGroupsHandler.updateCGroupParam(MEMORY, cgroupId, CGroupsHandler.CGROUP_PARAM_MEMORY_SOFT_LIMIT_BYTES, String.valueOf(containerSoftLimit) + "M");
        cGroupsHandler.updateCGroupParam(MEMORY, cgroupId, CGroupsHandler.CGROUP_PARAM_MEMORY_SWAPPINESS, String.valueOf(swappiness));
    } catch (ResourceHandlerException re) {
        cGroupsHandler.deleteCGroup(MEMORY, cgroupId);
        LOG.warn("Could not update cgroup for container", re);
        throw re;
    }
    List<PrivilegedOperation> ret = new ArrayList<>();
    ret.add(new PrivilegedOperation(PrivilegedOperation.OperationType.ADD_PID_TO_CGROUP, PrivilegedOperation.CGROUP_ARG_PREFIX + cGroupsHandler.getPathForCGroupTasks(MEMORY, cgroupId)));
    return ret;
}

Example 7 with PrivilegedOperation

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation in project hadoop by apache.

the class TestDockerContainerRuntime method testContainerLaunchWithCustomNetworks.

@Test
@SuppressWarnings("unchecked")
public void testContainerLaunchWithCustomNetworks() throws ContainerExecutionException, IOException, PrivilegedOperationException {
    DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
    String customNetwork1 = "sdn1";
    String customNetwork2 = "sdn2";
    String customNetwork3 = "sdn3";
    String[] networks = { "host", "none", "bridge", customNetwork1, customNetwork2 };
    //customized set of allowed networks
    conf.setStrings(YarnConfiguration.NM_DOCKER_ALLOWED_CONTAINER_NETWORKS, networks);
    //default network is "sdn1"
    conf.set(YarnConfiguration.NM_DOCKER_DEFAULT_CONTAINER_NETWORK, customNetwork1);
    //this should cause no failures.
    runtime.initialize(conf);
    runtime.launchContainer(builder.build());
    PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
    List<String> args = op.getArguments();
    String dockerCommandFile = args.get(11);
    //This is the expected docker invocation for this case. customNetwork1
    // ("sdn1") is the expected network to be used in this case
    StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ").append("--workdir=%3$s ").append("--net=" + customNetwork1 + " ").append(getExpectedTestCapabilitiesArgumentString()).append(getExpectedCGroupsMountString()).append("-v %4$s:%4$s ").append("-v %5$s:%5$s ").append("-v %6$s:%6$s ").append("-v %7$s:%7$s ").append("-v %8$s:%8$s ").append("%9$s ").append("bash %10$s/launch_container.sh");
    String expectedCommand = String.format(expectedCommandTemplate.toString(), containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), image, containerWorkDir);
    List<String> dockerCommands = Files.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
    Assert.assertEquals(1, dockerCommands.size());
    Assert.assertEquals(expectedCommand, dockerCommands.get(0));
    //now set an explicit (non-default) allowedNetwork and ensure that it is
    // used.
    env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", customNetwork2);
    runtime.launchContainer(builder.build());
    op = capturePrivilegedOperationAndVerifyArgs();
    args = op.getArguments();
    dockerCommandFile = args.get(11);
    //This is the expected docker invocation for this case. customNetwork2
    // ("sdn2") is the expected network to be used in this case
    expectedCommandTemplate = new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ").append("--workdir=%3$s ").append("--net=" + customNetwork2 + " ").append(getExpectedTestCapabilitiesArgumentString()).append(getExpectedCGroupsMountString()).append("-v %4$s:%4$s ").append("-v %5$s:%5$s ").append("-v %6$s:%6$s ").append("-v %7$s:%7$s ").append("-v %8$s:%8$s ").append("%9$s ").append("bash %10$s/launch_container.sh");
    expectedCommand = String.format(expectedCommandTemplate.toString(), containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), image, containerWorkDir);
    dockerCommands = Files.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
    Assert.assertEquals(1, dockerCommands.size());
    Assert.assertEquals(expectedCommand, dockerCommands.get(0));
    //disallowed network should trigger a launch failure
    env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", customNetwork3);
    try {
        runtime.launchContainer(builder.build());
        Assert.fail("Disallowed network : " + customNetwork3 + "did not trigger launch failure.");
    } catch (ContainerExecutionException e) {
        LOG.info("Caught expected exception : " + e);
    }
}

Example 8 with PrivilegedOperation

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation in project hadoop by apache.

the class TestDockerContainerRuntime method testContainerLaunchWithNetworkingDefaults.

@Test
@SuppressWarnings("unchecked")
public void testContainerLaunchWithNetworkingDefaults() throws ContainerExecutionException, IOException, PrivilegedOperationException {
    DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
    runtime.initialize(conf);
    Random randEngine = new Random();
    String disallowedNetwork = "sdn" + Integer.toString(randEngine.nextInt());
    try {
        env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", disallowedNetwork);
        runtime.launchContainer(builder.build());
        Assert.fail("Network was expected to be disallowed: " + disallowedNetwork);
    } catch (ContainerExecutionException e) {
        LOG.info("Caught expected exception: " + e);
    }
    int size = YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS.length;
    String allowedNetwork = YarnConfiguration.DEFAULT_NM_DOCKER_ALLOWED_CONTAINER_NETWORKS[randEngine.nextInt(size)];
    env.put("YARN_CONTAINER_RUNTIME_DOCKER_CONTAINER_NETWORK", allowedNetwork);
    //this should cause no failures.
    runtime.launchContainer(builder.build());
    PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
    List<String> args = op.getArguments();
    String dockerCommandFile = args.get(11);
    //This is the expected docker invocation for this case
    StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ").append("--workdir=%3$s ").append("--net=" + allowedNetwork + " ").append(getExpectedTestCapabilitiesArgumentString()).append(getExpectedCGroupsMountString()).append("-v %4$s:%4$s ").append("-v %5$s:%5$s ").append("-v %6$s:%6$s ").append("-v %7$s:%7$s ").append("-v %8$s:%8$s ").append("%9$s ").append("bash %10$s/launch_container.sh");
    String expectedCommand = String.format(expectedCommandTemplate.toString(), containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), image, containerWorkDir);
    List<String> dockerCommands = Files.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
    Assert.assertEquals(1, dockerCommands.size());
    Assert.assertEquals(expectedCommand, dockerCommands.get(0));
}

Example 9 with PrivilegedOperation

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation in project hadoop by apache.

the class TestDockerContainerRuntime method testDockerContainerLaunch.

@Test
public void testDockerContainerLaunch() throws ContainerExecutionException, PrivilegedOperationException, IOException {
    DockerLinuxContainerRuntime runtime = new DockerLinuxContainerRuntime(mockExecutor, mockCGroupsHandler);
    runtime.initialize(conf);
    runtime.launchContainer(builder.build());
    PrivilegedOperation op = capturePrivilegedOperationAndVerifyArgs();
    List<String> args = op.getArguments();
    String dockerCommandFile = args.get(11);
    //This is the expected docker invocation for this case
    StringBuffer expectedCommandTemplate = new StringBuffer("run --name=%1$s ").append("--user=%2$s -d ").append("--workdir=%3$s ").append("--net=host ").append(getExpectedTestCapabilitiesArgumentString()).append(getExpectedCGroupsMountString()).append("-v %4$s:%4$s ").append("-v %5$s:%5$s ").append("-v %6$s:%6$s ").append("-v %7$s:%7$s ").append("-v %8$s:%8$s ").append("%9$s ").append("bash %10$s/launch_container.sh");
    String expectedCommand = String.format(expectedCommandTemplate.toString(), containerId, runAsUser, containerWorkDir, containerLocalDirs.get(0), filecacheDirs.get(0), containerWorkDir, containerLogDirs.get(0), userLocalDirs.get(0), image, containerWorkDir);
    List<String> dockerCommands = Files.readAllLines(Paths.get(dockerCommandFile), Charset.forName("UTF-8"));
    Assert.assertEquals(1, dockerCommands.size());
    Assert.assertEquals(expectedCommand, dockerCommands.get(0));
}

Example 10 with PrivilegedOperation

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation in project hadoop by apache.

the class TestCGroupsBlkioResourceHandlerImpl method testBootstrap.

@Test
public void testBootstrap() throws Exception {
    Configuration conf = new YarnConfiguration();
    List<PrivilegedOperation> ret = cGroupsBlkioResourceHandlerImpl.bootstrap(conf);
    verify(mockCGroupsHandler, times(1)).initializeCGroupController(CGroupsHandler.CGroupController.BLKIO);
    Assert.assertNull(ret);
}