Examples with ContainerLaunchContext org.apache.hadoop.yarn.api.records.ContainerLaunchContext used on opensource projects

Example 1 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project flink by apache.

the class YarnResourceManager method createTaskExecutorLaunchContext.

private ContainerLaunchContext createTaskExecutorLaunchContext(Resource resource, String containerId, String host) throws Exception {
    // init the ContainerLaunchContext
    final String currDir = ENV.get(ApplicationConstants.Environment.PWD.key());
    final ContaineredTaskManagerParameters taskManagerParameters = ContaineredTaskManagerParameters.create(flinkConfig, resource.getMemory(), 1);
    LOG.info("TaskExecutor{} will be started with container size {} MB, JVM heap size {} MB, " + "JVM direct memory limit {} MB", containerId, taskManagerParameters.taskManagerTotalMemoryMB(), taskManagerParameters.taskManagerHeapSizeMB(), taskManagerParameters.taskManagerDirectMemoryLimitMB());
    int timeout = flinkConfig.getInteger(ConfigConstants.TASK_MANAGER_MAX_REGISTRATION_DURATION, DEFAULT_TASK_MANAGER_REGISTRATION_DURATION);
    FiniteDuration teRegistrationTimeout = new FiniteDuration(timeout, TimeUnit.SECONDS);
    final Configuration taskManagerConfig = BootstrapTools.generateTaskManagerConfiguration(flinkConfig, "", 0, 1, teRegistrationTimeout);
    LOG.debug("TaskManager configuration: {}", taskManagerConfig);
    ContainerLaunchContext taskExecutorLaunchContext = Utils.createTaskExecutorContext(flinkConfig, yarnConfig, ENV, taskManagerParameters, taskManagerConfig, currDir, YarnTaskExecutorRunner.class, LOG);
    // set a special environment variable to uniquely identify this container
    taskExecutorLaunchContext.getEnvironment().put(ENV_FLINK_CONTAINER_ID, containerId);
    taskExecutorLaunchContext.getEnvironment().put(ENV_FLINK_NODE_ID, host);
    return taskExecutorLaunchContext;
}

Example 2 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project flink by apache.

the class YarnResourceManager method onContainersAllocated.

@Override
public void onContainersAllocated(List<Container> containers) {
    for (Container container : containers) {
        numPendingContainerRequests = Math.max(0, numPendingContainerRequests - 1);
        LOG.info("Received new container: {} - Remaining pending container requests: {}", container.getId(), numPendingContainerRequests);
        try {
            /** Context information used to start a TaskExecutor Java process */
            ContainerLaunchContext taskExecutorLaunchContext = createTaskExecutorLaunchContext(container.getResource(), container.getId().toString(), container.getNodeId().getHost());
            nodeManagerClient.startContainer(container, taskExecutorLaunchContext);
        } catch (Throwable t) {
            // failed to launch the container, will release the failed one and ask for a new one
            LOG.error("Could not start TaskManager in container {},", container, t);
            resourceManagerClient.releaseAssignedContainer(container.getId());
            requestYarnContainer(container.getResource(), container.getPriority());
        }
    }
    if (numPendingContainerRequests <= 0) {
        resourceManagerClient.setHeartbeatInterval(yarnHeartbeatIntervalMillis);
    }
}

Example 3 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project flink by apache.

the class YarnApplicationMasterRunnerTest method testCreateTaskExecutorContext.

@Test
public void testCreateTaskExecutorContext() throws Exception {
    File root = folder.getRoot();
    File home = new File(root, "home");
    boolean created = home.mkdir();
    assertTrue(created);
    Answer<?> getDefault = new Answer<Object>() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            return invocationOnMock.getArguments()[1];
        }
    };
    Configuration flinkConf = new Configuration();
    YarnConfiguration yarnConf = mock(YarnConfiguration.class);
    doAnswer(getDefault).when(yarnConf).get(anyString(), anyString());
    doAnswer(getDefault).when(yarnConf).getInt(anyString(), anyInt());
    doAnswer(new Answer() {

        @Override
        public Object answer(InvocationOnMock invocationOnMock) throws Throwable {
            return new String[] { (String) invocationOnMock.getArguments()[1] };
        }
    }).when(yarnConf).getStrings(anyString(), Mockito.<String>anyVararg());
    Map<String, String> env = ImmutableMap.<String, String>builder().put(ENV_APP_ID, "foo").put(ENV_CLIENT_HOME_DIR, home.getAbsolutePath()).put(ENV_CLIENT_SHIP_FILES, "").put(ENV_FLINK_CLASSPATH, "").put(ENV_HADOOP_USER_NAME, "foo").put(FLINK_JAR_PATH, root.toURI().toString()).build();
    ContaineredTaskManagerParameters tmParams = mock(ContaineredTaskManagerParameters.class);
    Configuration taskManagerConf = new Configuration();
    String workingDirectory = root.getAbsolutePath();
    Class<?> taskManagerMainClass = YarnApplicationMasterRunnerTest.class;
    ContainerLaunchContext ctx = Utils.createTaskExecutorContext(flinkConf, yarnConf, env, tmParams, taskManagerConf, workingDirectory, taskManagerMainClass, LOG);
    assertEquals("file", ctx.getLocalResources().get("flink.jar").getResource().getScheme());
}

Example 4 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TaskAttemptImpl method createCommonContainerLaunchContext.

/**
   * Create the common {@link ContainerLaunchContext} for all attempts.
   * @param applicationACLs 
   */
private static ContainerLaunchContext createCommonContainerLaunchContext(Map<ApplicationAccessType, String> applicationACLs, Configuration conf, Token<JobTokenIdentifier> jobToken, final org.apache.hadoop.mapred.JobID oldJobId, Credentials credentials) {
    // Application resources
    Map<String, LocalResource> localResources = new HashMap<String, LocalResource>();
    // Application environment
    Map<String, String> environment = new HashMap<String, String>();
    // Service data
    Map<String, ByteBuffer> serviceData = new HashMap<String, ByteBuffer>();
    // Tokens
    ByteBuffer taskCredentialsBuffer = ByteBuffer.wrap(new byte[] {});
    try {
        FileSystem remoteFS = FileSystem.get(conf);
        // //////////// Set up JobJar to be localized properly on the remote NM.
        String jobJar = conf.get(MRJobConfig.JAR);
        if (jobJar != null) {
            final Path jobJarPath = new Path(jobJar);
            final FileSystem jobJarFs = FileSystem.get(jobJarPath.toUri(), conf);
            Path remoteJobJar = jobJarPath.makeQualified(jobJarFs.getUri(), jobJarFs.getWorkingDirectory());
            LocalResource rc = createLocalResource(jobJarFs, remoteJobJar, LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
            String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN, JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
            rc.setPattern(pattern);
            localResources.put(MRJobConfig.JOB_JAR, rc);
            LOG.info("The job-jar file on the remote FS is " + remoteJobJar.toUri().toASCIIString());
        } else {
            // Job jar may be null. For e.g, for pipes, the job jar is the hadoop
            // mapreduce jar itself which is already on the classpath.
            LOG.info("Job jar is not present. " + "Not adding any jar to the list of resources.");
        }
        // //////////// End of JobJar setup
        // //////////// Set up JobConf to be localized properly on the remote NM.
        Path path = MRApps.getStagingAreaDir(conf, UserGroupInformation.getCurrentUser().getShortUserName());
        Path remoteJobSubmitDir = new Path(path, oldJobId.toString());
        Path remoteJobConfPath = new Path(remoteJobSubmitDir, MRJobConfig.JOB_CONF_FILE);
        localResources.put(MRJobConfig.JOB_CONF_FILE, createLocalResource(remoteFS, remoteJobConfPath, LocalResourceType.FILE, LocalResourceVisibility.APPLICATION));
        LOG.info("The job-conf file on the remote FS is " + remoteJobConfPath.toUri().toASCIIString());
        // //////////// End of JobConf setup
        // Setup DistributedCache
        MRApps.setupDistributedCache(conf, localResources);
        // Setup up task credentials buffer
        LOG.info("Adding #" + credentials.numberOfTokens() + " tokens and #" + credentials.numberOfSecretKeys() + " secret keys for NM use for launching container");
        Credentials taskCredentials = new Credentials(credentials);
        // LocalStorageToken is needed irrespective of whether security is enabled
        // or not.
        TokenCache.setJobToken(jobToken, taskCredentials);
        DataOutputBuffer containerTokens_dob = new DataOutputBuffer();
        LOG.info("Size of containertokens_dob is " + taskCredentials.numberOfTokens());
        taskCredentials.writeTokenStorageToStream(containerTokens_dob);
        taskCredentialsBuffer = ByteBuffer.wrap(containerTokens_dob.getData(), 0, containerTokens_dob.getLength());
        // Add shuffle secret key
        // The secret key is converted to a JobToken to preserve backwards
        // compatibility with an older ShuffleHandler running on an NM.
        LOG.info("Putting shuffle token in serviceData");
        byte[] shuffleSecret = TokenCache.getShuffleSecretKey(credentials);
        if (shuffleSecret == null) {
            LOG.warn("Cannot locate shuffle secret in credentials." + " Using job token as shuffle secret.");
            shuffleSecret = jobToken.getPassword();
        }
        Token<JobTokenIdentifier> shuffleToken = new Token<JobTokenIdentifier>(jobToken.getIdentifier(), shuffleSecret, jobToken.getKind(), jobToken.getService());
        serviceData.put(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID, ShuffleHandler.serializeServiceData(shuffleToken));
        // add external shuffle-providers - if any
        Collection<String> shuffleProviders = conf.getStringCollection(MRJobConfig.MAPREDUCE_JOB_SHUFFLE_PROVIDER_SERVICES);
        if (!shuffleProviders.isEmpty()) {
            Collection<String> auxNames = conf.getStringCollection(YarnConfiguration.NM_AUX_SERVICES);
            for (final String shuffleProvider : shuffleProviders) {
                if (shuffleProvider.equals(ShuffleHandler.MAPREDUCE_SHUFFLE_SERVICEID)) {
                    // skip built-in shuffle-provider that was already inserted with shuffle secret key
                    continue;
                }
                if (auxNames.contains(shuffleProvider)) {
                    LOG.info("Adding ShuffleProvider Service: " + shuffleProvider + " to serviceData");
                    // This only serves for INIT_APP notifications
                    // The shuffle service needs to be able to work with the host:port information provided by the AM
                    // (i.e. shuffle services which require custom location / other configuration are not supported)
                    serviceData.put(shuffleProvider, ByteBuffer.allocate(0));
                } else {
                    throw new YarnRuntimeException("ShuffleProvider Service: " + shuffleProvider + " was NOT found in the list of aux-services that are available in this NM." + " You may need to specify this ShuffleProvider as an aux-service in your yarn-site.xml");
                }
            }
        }
        MRApps.addToEnvironment(environment, Environment.CLASSPATH.name(), getInitialClasspath(conf), conf);
        if (initialAppClasspath != null) {
            MRApps.addToEnvironment(environment, Environment.APP_CLASSPATH.name(), initialAppClasspath, conf);
        }
    } catch (IOException e) {
        throw new YarnRuntimeException(e);
    }
    // Shell
    environment.put(Environment.SHELL.name(), conf.get(MRJobConfig.MAPRED_ADMIN_USER_SHELL, MRJobConfig.DEFAULT_SHELL));
    // Add pwd to LD_LIBRARY_PATH, add this before adding anything else
    MRApps.addToEnvironment(environment, Environment.LD_LIBRARY_PATH.name(), MRApps.crossPlatformifyMREnv(conf, Environment.PWD), conf);
    // Add the env variables passed by the admin
    MRApps.setEnvFromInputString(environment, conf.get(MRJobConfig.MAPRED_ADMIN_USER_ENV, MRJobConfig.DEFAULT_MAPRED_ADMIN_USER_ENV), conf);
    // Construct the actual Container
    // The null fields are per-container and will be constructed for each
    // container separately.
    ContainerLaunchContext container = ContainerLaunchContext.newInstance(localResources, environment, null, serviceData, taskCredentialsBuffer, applicationACLs);
    return container;
}

Example 5 with ContainerLaunchContext

use of org.apache.hadoop.yarn.api.records.ContainerLaunchContext in project hadoop by apache.

the class TestNMClientAsync method testOutOfOrder.

@Test(timeout = 10000)
public void testOutOfOrder() throws Exception {
    CyclicBarrier barrierA = new CyclicBarrier(2);
    CyclicBarrier barrierB = new CyclicBarrier(2);
    CyclicBarrier barrierC = new CyclicBarrier(2);
    asyncClient = new MockNMClientAsync2(barrierA, barrierB, barrierC);
    asyncClient.init(new Configuration());
    asyncClient.start();
    final Container container = mockContainer(1);
    final ContainerLaunchContext clc = recordFactory.newRecordInstance(ContainerLaunchContext.class);
    // start container from another thread
    Thread t = new Thread() {

        @Override
        public void run() {
            asyncClient.startContainerAsync(container, clc);
        }
    };
    t.start();
    barrierA.await();
    asyncClient.stopContainerAsync(container.getId(), container.getNodeId());
    barrierC.await();
    Assert.assertFalse("Starting and stopping should be out of order", ((TestCallbackHandler2) asyncClient.getCallbackHandler()).exceptionOccurred.get());
}