Examples with ContainerLaunchContext org.apache.hadoop.yarn.api.records.ContainerLaunchContext used on opensource projects

Example 96 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestContainerLaunchRPC method testRPCTimeout.

private void testRPCTimeout(String rpcClass) throws Exception {
    Configuration conf = new Configuration();
    // set timeout low for the test
    conf.setInt("yarn.rpc.nm-command-timeout", 3000);
    conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1);
    server.start();
    try {
        ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, server.getListenerAddress(), conf);
        ContainerLaunchContext containerLaunchContext = recordFactory.newRecordInstance(ContainerLaunchContext.class);
        ApplicationId applicationId = ApplicationId.newInstance(0, 0);
        ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
        ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 100);
        NodeId nodeId = NodeId.newInstance("localhost", 1234);
        Resource resource = Resource.newInstance(1234, 2);
        ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0);
        Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier);
        StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
        List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
        list.add(scRequest);
        StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
        try {
            proxy.startContainers(allRequests);
        } catch (Exception e) {
            LOG.info(StringUtils.stringifyException(e));
            Assert.assertEquals("Error, exception is not: " + SocketTimeoutException.class.getName(), SocketTimeoutException.class.getName(), e.getClass().getName());
            return;
        }
    } finally {
        server.stop();
    }
    Assert.fail("timeout exception should have occurred!");
}

Example 97 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestRPC method test.

private void test(String rpcClass) throws Exception {
    Configuration conf = new Configuration();
    conf.set(YarnConfiguration.IPC_RPC_IMPL, rpcClass);
    YarnRPC rpc = YarnRPC.create(conf);
    String bindAddr = "localhost:0";
    InetSocketAddress addr = NetUtils.createSocketAddr(bindAddr);
    Server server = rpc.getServer(ContainerManagementProtocol.class, new DummyContainerManager(), addr, conf, null, 1);
    server.start();
    RPC.setProtocolEngine(conf, ContainerManagementProtocolPB.class, ProtobufRpcEngine.class);
    ContainerManagementProtocol proxy = (ContainerManagementProtocol) rpc.getProxy(ContainerManagementProtocol.class, NetUtils.getConnectAddress(server), conf);
    ContainerLaunchContext containerLaunchContext = RECORD_FACTORY.newRecordInstance(ContainerLaunchContext.class);
    ApplicationId applicationId = ApplicationId.newInstance(0, 0);
    ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(applicationId, 0);
    ContainerId containerId = ContainerId.newContainerId(applicationAttemptId, 100);
    NodeId nodeId = NodeId.newInstance("localhost", 1234);
    Resource resource = Resource.newInstance(1234, 2);
    ContainerTokenIdentifier containerTokenIdentifier = new ContainerTokenIdentifier(containerId, "localhost", "user", resource, System.currentTimeMillis() + 10000, 42, 42, Priority.newInstance(0), 0);
    Token containerToken = newContainerToken(nodeId, "password".getBytes(), containerTokenIdentifier);
    StartContainerRequest scRequest = StartContainerRequest.newInstance(containerLaunchContext, containerToken);
    List<StartContainerRequest> list = new ArrayList<StartContainerRequest>();
    list.add(scRequest);
    StartContainersRequest allRequests = StartContainersRequest.newInstance(list);
    proxy.startContainers(allRequests);
    List<ContainerId> containerIds = new ArrayList<ContainerId>();
    containerIds.add(containerId);
    GetContainerStatusesRequest gcsRequest = GetContainerStatusesRequest.newInstance(containerIds);
    GetContainerStatusesResponse response = proxy.getContainerStatuses(gcsRequest);
    List<ContainerStatus> statuses = response.getContainerStatuses();
    //test remote exception
    boolean exception = false;
    try {
        StopContainersRequest stopRequest = RECORD_FACTORY.newRecordInstance(StopContainersRequest.class);
        stopRequest.setContainerIds(containerIds);
        proxy.stopContainers(stopRequest);
    } catch (YarnException e) {
        exception = true;
        Assert.assertTrue(e.getMessage().contains(EXCEPTION_MSG));
        Assert.assertTrue(e.getMessage().contains(EXCEPTION_CAUSE));
        System.out.println("Test Exception is " + e.getMessage());
    } catch (Exception ex) {
        ex.printStackTrace();
    } finally {
        server.stop();
    }
    Assert.assertTrue(exception);
    Assert.assertNotNull(statuses.get(0));
    Assert.assertEquals(ContainerState.RUNNING, statuses.get(0).getState());
}

Example 98 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class ContainerManagerImpl method recoverContainer.

private void recoverContainer(RecoveredContainerState rcs) throws IOException {
    StartContainerRequest req = rcs.getStartRequest();
    ContainerLaunchContext launchContext = req.getContainerLaunchContext();
    ContainerTokenIdentifier token = BuilderUtils.newContainerTokenIdentifier(req.getContainerToken());
    ContainerId containerId = token.getContainerID();
    ApplicationId appId = containerId.getApplicationAttemptId().getApplicationId();
    LOG.info("Recovering " + containerId + " in state " + rcs.getStatus() + " with exit code " + rcs.getExitCode());
    Application app = context.getApplications().get(appId);
    if (app != null) {
        recoverActiveContainer(app, launchContext, token, rcs);
        if (rcs.getRecoveryType() == RecoveredContainerType.KILL) {
            dispatcher.getEventHandler().handle(new ContainerKillEvent(containerId, ContainerExitStatus.ABORTED, "Due to invalid StateStore info container was killed" + " during recovery"));
        }
    } else {
        if (rcs.getStatus() != RecoveredContainerStatus.COMPLETED) {
            LOG.warn(containerId + " has no corresponding application!");
        }
        LOG.info("Adding " + containerId + " to recently stopped containers");
        nodeStatusUpdater.addCompletedContainer(containerId);
    }
}

Example 99 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class ContainerLaunch method call.

@Override
// dispatcher not typed
@SuppressWarnings("unchecked")
public Integer call() {
    if (!validateContainerState()) {
        return 0;
    }
    final ContainerLaunchContext launchContext = container.getLaunchContext();
    ContainerId containerID = container.getContainerId();
    String containerIdStr = containerID.toString();
    final List<String> command = launchContext.getCommands();
    int ret = -1;
    Path containerLogDir;
    try {
        Map<Path, List<String>> localResources = getLocalizedResources();
        final String user = container.getUser();
        // /////////////////////////// Variable expansion
        // Before the container script gets written out.
        List<String> newCmds = new ArrayList<String>(command.size());
        String appIdStr = app.getAppId().toString();
        String relativeContainerLogDir = ContainerLaunch.getRelativeContainerLogDir(appIdStr, containerIdStr);
        containerLogDir = dirsHandler.getLogPathForWrite(relativeContainerLogDir, false);
        recordContainerLogDir(containerID, containerLogDir.toString());
        for (String str : command) {
            // TODO: Should we instead work via symlinks without this grammar?
            newCmds.add(expandEnvironment(str, containerLogDir));
        }
        launchContext.setCommands(newCmds);
        Map<String, String> environment = launchContext.getEnvironment();
        // Make a copy of env to iterate & do variable expansion
        for (Entry<String, String> entry : environment.entrySet()) {
            String value = entry.getValue();
            value = expandEnvironment(value, containerLogDir);
            entry.setValue(value);
        }
        // /////////////////////////// End of variable expansion
        FileContext lfs = FileContext.getLocalFSFileContext();
        Path nmPrivateContainerScriptPath = dirsHandler.getLocalPathForWrite(getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR + CONTAINER_SCRIPT);
        Path nmPrivateTokensPath = dirsHandler.getLocalPathForWrite(getContainerPrivateDir(appIdStr, containerIdStr) + Path.SEPARATOR + String.format(ContainerLocalizer.TOKEN_FILE_NAME_FMT, containerIdStr));
        Path nmPrivateClasspathJarDir = dirsHandler.getLocalPathForWrite(getContainerPrivateDir(appIdStr, containerIdStr));
        DataOutputStream containerScriptOutStream = null;
        DataOutputStream tokensOutStream = null;
        // Select the working directory for the container
        Path containerWorkDir = dirsHandler.getLocalPathForWrite(ContainerLocalizer.USERCACHE + Path.SEPARATOR + user + Path.SEPARATOR + ContainerLocalizer.APPCACHE + Path.SEPARATOR + appIdStr + Path.SEPARATOR + containerIdStr, LocalDirAllocator.SIZE_UNKNOWN, false);
        recordContainerWorkDir(containerID, containerWorkDir.toString());
        String pidFileSubpath = getPidFileSubpath(appIdStr, containerIdStr);
        // pid file should be in nm private dir so that it is not 
        // accessible by users
        pidFilePath = dirsHandler.getLocalPathForWrite(pidFileSubpath);
        List<String> localDirs = dirsHandler.getLocalDirs();
        List<String> logDirs = dirsHandler.getLogDirs();
        List<String> filecacheDirs = getNMFilecacheDirs(localDirs);
        List<String> userLocalDirs = getUserLocalDirs(localDirs);
        List<String> containerLocalDirs = getContainerLocalDirs(localDirs);
        List<String> containerLogDirs = getContainerLogDirs(logDirs);
        if (!dirsHandler.areDisksHealthy()) {
            ret = ContainerExitStatus.DISKS_FAILED;
            throw new IOException("Most of the disks failed. " + dirsHandler.getDisksHealthReport(false));
        }
        try {
            // /////////// Write out the container-script in the nmPrivate space.
            List<Path> appDirs = new ArrayList<Path>(localDirs.size());
            for (String localDir : localDirs) {
                Path usersdir = new Path(localDir, ContainerLocalizer.USERCACHE);
                Path userdir = new Path(usersdir, user);
                Path appsdir = new Path(userdir, ContainerLocalizer.APPCACHE);
                appDirs.add(new Path(appsdir, appIdStr));
            }
            containerScriptOutStream = lfs.create(nmPrivateContainerScriptPath, EnumSet.of(CREATE, OVERWRITE));
            // Set the token location too.
            environment.put(ApplicationConstants.CONTAINER_TOKEN_FILE_ENV_NAME, new Path(containerWorkDir, FINAL_CONTAINER_TOKENS_FILE).toUri().getPath());
            // Sanitize the container's environment
            sanitizeEnv(environment, containerWorkDir, appDirs, userLocalDirs, containerLogDirs, localResources, nmPrivateClasspathJarDir);
            exec.prepareContainer(new ContainerPrepareContext.Builder().setContainer(container).setLocalizedResources(localResources).setUser(user).setContainerLocalDirs(containerLocalDirs).setCommands(launchContext.getCommands()).build());
            // Write out the environment
            exec.writeLaunchEnv(containerScriptOutStream, environment, localResources, launchContext.getCommands(), new Path(containerLogDirs.get(0)), user);
            // /////////// End of writing out container-script
            // /////////// Write out the container-tokens in the nmPrivate space.
            tokensOutStream = lfs.create(nmPrivateTokensPath, EnumSet.of(CREATE, OVERWRITE));
            Credentials creds = container.getCredentials();
            creds.writeTokenStorageToStream(tokensOutStream);
        // /////////// End of writing out container-tokens
        } finally {
            IOUtils.cleanup(LOG, containerScriptOutStream, tokensOutStream);
        }
        ret = launchContainer(new ContainerStartContext.Builder().setContainer(container).setLocalizedResources(localResources).setNmPrivateContainerScriptPath(nmPrivateContainerScriptPath).setNmPrivateTokensPath(nmPrivateTokensPath).setUser(user).setAppId(appIdStr).setContainerWorkDir(containerWorkDir).setLocalDirs(localDirs).setLogDirs(logDirs).setFilecacheDirs(filecacheDirs).setUserLocalDirs(userLocalDirs).setContainerLocalDirs(containerLocalDirs).setContainerLogDirs(containerLogDirs).build());
    } catch (Throwable e) {
        LOG.warn("Failed to launch container.", e);
        dispatcher.getEventHandler().handle(new ContainerExitEvent(containerID, ContainerEventType.CONTAINER_EXITED_WITH_FAILURE, ret, e.getMessage()));
        return ret;
    } finally {
        setContainerCompletedStatus(ret);
    }
    handleContainerExitCode(ret, containerLogDir);
    return ret;
}

Example 100 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestLinuxContainerExecutorWithMocks method testContainerLaunchError.

@Test
public void testContainerLaunchError() throws IOException, ContainerExecutionException {
    // reinitialize executer
    Configuration conf = new Configuration();
    setupMockExecutor(MOCK_EXECUTOR_WITH_ERROR, conf);
    conf.set(YarnConfiguration.NM_LOCAL_DIRS, "file:///bin/echo");
    conf.set(YarnConfiguration.NM_LOG_DIRS, "file:///dev/null");
    LinuxContainerExecutor exec;
    LinuxContainerRuntime linuxContainerRuntime = new DefaultLinuxContainerRuntime(PrivilegedOperationExecutor.getInstance(conf));
    linuxContainerRuntime.initialize(conf);
    exec = new LinuxContainerExecutor(linuxContainerRuntime);
    mockExec = spy(exec);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            String diagnostics = (String) invocationOnMock.getArguments()[0];
            assertTrue("Invalid Diagnostics message: " + diagnostics, diagnostics.contains("badcommand"));
            return null;
        }
    }).when(mockExec).logOutput(any(String.class));
    dirsHandler = new LocalDirsHandlerService();
    dirsHandler.init(conf);
    mockExec.setConf(conf);
    String appSubmitter = "nobody";
    String cmd = String.valueOf(PrivilegedOperation.RunAsUserCommand.LAUNCH_CONTAINER.getValue());
    String appId = "APP_ID";
    String containerId = "CONTAINER_ID";
    Container container = mock(Container.class);
    ContainerId cId = mock(ContainerId.class);
    ContainerLaunchContext context = mock(ContainerLaunchContext.class);
    HashMap<String, String> env = new HashMap<String, String>();
    when(container.getContainerId()).thenReturn(cId);
    when(container.getLaunchContext()).thenReturn(context);
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            ContainerDiagnosticsUpdateEvent event = (ContainerDiagnosticsUpdateEvent) invocationOnMock.getArguments()[0];
            assertTrue("Invalid Diagnostics message: " + event.getDiagnosticsUpdate(), event.getDiagnosticsUpdate().contains("badcommand"));
            return null;
        }
    }).when(container).handle(any(ContainerDiagnosticsUpdateEvent.class));
    when(cId.toString()).thenReturn(containerId);
    when(context.getEnvironment()).thenReturn(env);
    Path scriptPath = new Path("file:///bin/echo");
    Path tokensPath = new Path("file:///dev/null");
    Path workDir = new Path("/tmp");
    Path pidFile = new Path(workDir, "pid.txt");
    mockExec.activateContainer(cId, pidFile);
    int ret = mockExec.launchContainer(new ContainerStartContext.Builder().setContainer(container).setNmPrivateContainerScriptPath(scriptPath).setNmPrivateTokensPath(tokensPath).setUser(appSubmitter).setAppId(appId).setContainerWorkDir(workDir).setLocalDirs(dirsHandler.getLocalDirs()).setLogDirs(dirsHandler.getLogDirs()).setFilecacheDirs(new ArrayList<>()).setUserLocalDirs(new ArrayList<>()).setContainerLocalDirs(new ArrayList<>()).setContainerLogDirs(new ArrayList<>()).build());
    Assert.assertNotSame(0, ret);
    assertEquals(Arrays.asList(YarnConfiguration.DEFAULT_NM_NONSECURE_MODE_LOCAL_USER, appSubmitter, cmd, appId, containerId, workDir.toString(), "/bin/echo", "/dev/null", pidFile.toString(), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, dirsHandler.getLocalDirs()), StringUtils.join(PrivilegedOperation.LINUX_FILE_PATH_SEPARATOR, dirsHandler.getLogDirs()), "cgroups=none"), readMockParams());
}