Examples with Container org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container used on opensource projects

Example 91 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestContainerLaunch method verifyTailErrorLogOnContainerExit.

private void verifyTailErrorLogOnContainerExit(Configuration conf, String errorFileName, boolean testForMultipleErrFiles) throws Exception {
    Container container = mock(Container.class);
    ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
    ContainerId containerId = ContainerId.newContainerId(ApplicationAttemptId.newInstance(appId, 1), 1);
    when(container.getContainerId()).thenReturn(containerId);
    when(container.getUser()).thenReturn("test");
    String relativeContainerLogDir = ContainerLaunch.getRelativeContainerLogDir(appId.toString(), containerId.toString());
    Path containerLogDir = dirsHandler.getLogPathForWrite(relativeContainerLogDir, false);
    ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
    List<String> invalidCommand = new ArrayList<String>();
    invalidCommand.add("$JAVA_HOME/bin/java");
    invalidCommand.add("-Djava.io.tmpdir=$PWD/tmp");
    invalidCommand.add("-Dlog4j.configuration=container-log4j.properties");
    invalidCommand.add("-Dyarn.app.container.log.dir=" + containerLogDir);
    invalidCommand.add("-Dyarn.app.container.log.filesize=0");
    invalidCommand.add("-Dhadoop.root.logger=INFO,CLA");
    invalidCommand.add("-Dhadoop.root.logfile=syslog");
    invalidCommand.add("-Xmx1024m");
    invalidCommand.add("org.apache.hadoop.mapreduce.v2.app.MRAppMaster");
    invalidCommand.add("1>" + containerLogDir + "/stdout");
    invalidCommand.add("2>" + containerLogDir + errorFileName);
    when(clc.getCommands()).thenReturn(invalidCommand);
    Map<String, String> userSetEnv = new HashMap<String, String>();
    userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id");
    userSetEnv.put("JAVA_HOME", INVALID_JAVA_HOME);
    userSetEnv.put(Environment.NM_HOST.name(), "user_set_NM_HOST");
    userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT");
    userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT");
    userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR");
    userSetEnv.put(Environment.USER.key(), "user_set_" + Environment.USER.key());
    userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
    userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
    userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
    userSetEnv.put(Environment.CLASSPATH.name(), "APATH");
    when(clc.getEnvironment()).thenReturn(userSetEnv);
    when(container.getLaunchContext()).thenReturn(clc);
    when(container.getLocalizedResources()).thenReturn(Collections.<Path, List<String>>emptyMap());
    Dispatcher dispatcher = mock(Dispatcher.class);
    @SuppressWarnings("rawtypes") ContainerExitHandler eventHandler = new ContainerExitHandler(testForMultipleErrFiles);
    when(dispatcher.getEventHandler()).thenReturn(eventHandler);
    Application app = mock(Application.class);
    when(app.getAppId()).thenReturn(appId);
    when(app.getUser()).thenReturn("test");
    Credentials creds = mock(Credentials.class);
    when(container.getCredentials()).thenReturn(creds);
    ((NMContext) context).setNodeId(NodeId.newInstance("127.0.0.1", HTTP_PORT));
    ContainerLaunch launch = new ContainerLaunch(context, conf, dispatcher, exec, app, container, dirsHandler, containerManager);
    launch.call();
    Assert.assertTrue("ContainerExitEvent should have occured", eventHandler.isContainerExitEventOccured());
}

Example 92 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestContainerLaunch method testCallFailureWithNullLocalizedResources.

@SuppressWarnings("rawtypes")
@Test(timeout = 10000)
public void testCallFailureWithNullLocalizedResources() {
    Container container = mock(Container.class);
    when(container.getContainerId()).thenReturn(ContainerId.newContainerId(ApplicationAttemptId.newInstance(ApplicationId.newInstance(System.currentTimeMillis(), 1), 1), 1));
    ContainerLaunchContext clc = mock(ContainerLaunchContext.class);
    when(clc.getCommands()).thenReturn(Collections.<String>emptyList());
    when(container.getLaunchContext()).thenReturn(clc);
    when(container.getLocalizedResources()).thenReturn(null);
    Dispatcher dispatcher = mock(Dispatcher.class);
    EventHandler<Event> eventHandler = new EventHandler<Event>() {

        @Override
        public void handle(Event event) {
            Assert.assertTrue(event instanceof ContainerExitEvent);
            ContainerExitEvent exitEvent = (ContainerExitEvent) event;
            Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, exitEvent.getType());
        }
    };
    when(dispatcher.getEventHandler()).thenReturn(eventHandler);
    ContainerLaunch launch = new ContainerLaunch(context, new Configuration(), dispatcher, exec, null, container, dirsHandler, containerManager);
    launch.call();
}

Example 93 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestContainerLaunch method testPrependDistcache.

@Test
public void testPrependDistcache() throws Exception {
    // Test is only relevant on Windows
    assumeWindows();
    ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    ApplicationId appId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId cId = ContainerId.newContainerId(appAttemptId, 0);
    Map<String, String> userSetEnv = new HashMap<String, String>();
    userSetEnv.put(Environment.CONTAINER_ID.name(), "user_set_container_id");
    userSetEnv.put(Environment.NM_HOST.name(), "user_set_NM_HOST");
    userSetEnv.put(Environment.NM_PORT.name(), "user_set_NM_PORT");
    userSetEnv.put(Environment.NM_HTTP_PORT.name(), "user_set_NM_HTTP_PORT");
    userSetEnv.put(Environment.LOCAL_DIRS.name(), "user_set_LOCAL_DIR");
    userSetEnv.put(Environment.USER.key(), "user_set_" + Environment.USER.key());
    userSetEnv.put(Environment.LOGNAME.name(), "user_set_LOGNAME");
    userSetEnv.put(Environment.PWD.name(), "user_set_PWD");
    userSetEnv.put(Environment.HOME.name(), "user_set_HOME");
    userSetEnv.put(Environment.CLASSPATH.name(), "APATH");
    containerLaunchContext.setEnvironment(userSetEnv);
    Container container = mock(Container.class);
    when(container.getContainerId()).thenReturn(cId);
    when(container.getLaunchContext()).thenReturn(containerLaunchContext);
    when(container.getLocalizedResources()).thenReturn(null);
    Dispatcher dispatcher = mock(Dispatcher.class);
    EventHandler<Event> eventHandler = new EventHandler<Event>() {

        public void handle(Event event) {
            Assert.assertTrue(event instanceof ContainerExitEvent);
            ContainerExitEvent exitEvent = (ContainerExitEvent) event;
            Assert.assertEquals(ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, exitEvent.getType());
        }
    };
    when(dispatcher.getEventHandler()).thenReturn(eventHandler);
    Configuration conf = new Configuration();
    ContainerLaunch launch = new ContainerLaunch(distContext, conf, dispatcher, exec, null, container, dirsHandler, containerManager);
    String testDir = System.getProperty("test.build.data", "target/test-dir");
    Path pwd = new Path(testDir);
    List<Path> appDirs = new ArrayList<Path>();
    List<String> userLocalDirs = new ArrayList<>();
    List<String> containerLogs = new ArrayList<String>();
    Map<Path, List<String>> resources = new HashMap<Path, List<String>>();
    Path userjar = new Path("user.jar");
    List<String> lpaths = new ArrayList<String>();
    lpaths.add("userjarlink.jar");
    resources.put(userjar, lpaths);
    Path nmp = new Path(testDir);
    launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs, resources, nmp);
    List<String> result = getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));
    Assert.assertTrue(result.size() > 1);
    Assert.assertTrue(result.get(result.size() - 1).endsWith("userjarlink.jar"));
    //Then, with user classpath first
    userSetEnv.put(Environment.CLASSPATH_PREPEND_DISTCACHE.name(), "true");
    cId = ContainerId.newContainerId(appAttemptId, 1);
    when(container.getContainerId()).thenReturn(cId);
    launch = new ContainerLaunch(distContext, conf, dispatcher, exec, null, container, dirsHandler, containerManager);
    launch.sanitizeEnv(userSetEnv, pwd, appDirs, userLocalDirs, containerLogs, resources, nmp);
    result = getJarManifestClasspath(userSetEnv.get(Environment.CLASSPATH.name()));
    Assert.assertTrue(result.size() > 1);
    Assert.assertTrue(result.get(0).endsWith("userjarlink.jar"));
}

Example 94 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestApplication method testAppFinishedOnCompletedContainers.

@Test
@SuppressWarnings("unchecked")
public void testAppFinishedOnCompletedContainers() {
    WrappedApplication wa = null;
    try {
        wa = new WrappedApplication(5, 314159265358979L, "yak", 3);
        wa.initApplication();
        wa.initContainer(-1);
        assertEquals(ApplicationState.INITING, wa.app.getApplicationState());
        wa.applicationInited();
        assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
        reset(wa.localizerBus);
        wa.containerFinished(0);
        wa.containerFinished(1);
        wa.containerFinished(2);
        assertEquals(ApplicationState.RUNNING, wa.app.getApplicationState());
        assertEquals(0, wa.app.getContainers().size());
        wa.appFinished();
        assertEquals(ApplicationState.APPLICATION_RESOURCES_CLEANINGUP, wa.app.getApplicationState());
        verify(wa.localizerBus).handle(refEq(new ApplicationLocalizationEvent(LocalizationEventType.DESTROY_APPLICATION_RESOURCES, wa.app)));
        wa.appResourcesCleanedup();
        for (Container container : wa.containers) {
            ContainerTokenIdentifier identifier = wa.getContainerTokenIdentifier(container.getContainerId());
            waitForContainerTokenToExpire(identifier);
            Assert.assertTrue(wa.context.getContainerTokenSecretManager().isValidStartContainerRequest(identifier));
        }
        assertEquals(ApplicationState.FINISHED, wa.app.getApplicationState());
    } finally {
        if (wa != null)
            wa.finished();
    }
}

Example 95 with Container

use of org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container in project hadoop by apache.

the class TestDockerContainerRuntime method setup.

@Before
public void setup() {
    String tmpPath = new StringBuffer(System.getProperty("test.build.data")).append('/').append("hadoop.tmp.dir").toString();
    conf = new Configuration();
    conf.set("hadoop.tmp.dir", tmpPath);
    mockExecutor = Mockito.mock(PrivilegedOperationExecutor.class);
    mockCGroupsHandler = Mockito.mock(CGroupsHandler.class);
    containerId = "container_id";
    container = mock(Container.class);
    cId = mock(ContainerId.class);
    context = mock(ContainerLaunchContext.class);
    env = new HashMap<String, String>();
    image = "busybox:latest";
    env.put(DockerLinuxContainerRuntime.ENV_DOCKER_CONTAINER_IMAGE, image);
    when(container.getContainerId()).thenReturn(cId);
    when(cId.toString()).thenReturn(containerId);
    when(container.getLaunchContext()).thenReturn(context);
    when(context.getEnvironment()).thenReturn(env);
    when(container.getUser()).thenReturn(submittingUser);
    runAsUser = "run_as_user";
    user = "user";
    appId = "app_id";
    containerIdStr = containerId;
    containerWorkDir = new Path("/test_container_work_dir");
    nmPrivateContainerScriptPath = new Path("/test_script_path");
    nmPrivateTokensPath = new Path("/test_private_tokens_path");
    pidFilePath = new Path("/test_pid_file_path");
    localDirs = new ArrayList<>();
    logDirs = new ArrayList<>();
    filecacheDirs = new ArrayList<>();
    resourcesOptions = "cgroups=none";
    userLocalDirs = new ArrayList<>();
    containerLocalDirs = new ArrayList<>();
    containerLogDirs = new ArrayList<>();
    localizedResources = new HashMap<>();
    localDirs.add("/test_local_dir");
    logDirs.add("/test_log_dir");
    filecacheDirs.add("/test_filecache_dir");
    userLocalDirs.add("/test_user_local_dir");
    containerLocalDirs.add("/test_container_local_dir");
    containerLogDirs.add("/test_container_log_dir");
    localizedResources.put(new Path("/test_local_dir/test_resource_file"), Collections.singletonList("test_dir/test_resource_file"));
    testCapabilities = new String[] { "NET_BIND_SERVICE", "SYS_CHROOT" };
    conf.setStrings(YarnConfiguration.NM_DOCKER_CONTAINER_CAPABILITIES, testCapabilities);
    builder = new ContainerRuntimeContext.Builder(container);
    builder.setExecutionAttribute(RUN_AS_USER, runAsUser).setExecutionAttribute(USER, user).setExecutionAttribute(APPID, appId).setExecutionAttribute(CONTAINER_ID_STR, containerIdStr).setExecutionAttribute(CONTAINER_WORK_DIR, containerWorkDir).setExecutionAttribute(NM_PRIVATE_CONTAINER_SCRIPT_PATH, nmPrivateContainerScriptPath).setExecutionAttribute(NM_PRIVATE_TOKENS_PATH, nmPrivateTokensPath).setExecutionAttribute(PID_FILE_PATH, pidFilePath).setExecutionAttribute(LOCAL_DIRS, localDirs).setExecutionAttribute(LOG_DIRS, logDirs).setExecutionAttribute(FILECACHE_DIRS, filecacheDirs).setExecutionAttribute(USER_LOCAL_DIRS, userLocalDirs).setExecutionAttribute(CONTAINER_LOCAL_DIRS, containerLocalDirs).setExecutionAttribute(CONTAINER_LOG_DIRS, containerLogDirs).setExecutionAttribute(LOCALIZED_RESOURCES, localizedResources).setExecutionAttribute(RESOURCES_OPTIONS, resourcesOptions);
}