Examples with ContainerExecutionResult org.apache.tez.runtime.task.TezChild.ContainerExecutionResult used on opensource projects

Example 1 with ContainerExecutionResult

use of org.apache.tez.runtime.task.TezChild.ContainerExecutionResult in project tez by apache.

the class ContainerRunnerImpl method submitWork.

/**
 * Submit an entire work unit - containerId + TaskSpec.
 * This is intended for a task push from the AM
 *
 * @param request
 * @throws org.apache.tez.dag.api.TezException
 */
@Override
public void submitWork(SubmitWorkRequestProto request) throws TezException {
    LOG.info("Queuing work for execution: " + request);
    checkAndThrowExceptionForTests(request);
    Map<String, String> env = new HashMap<String, String>();
    env.putAll(localEnv);
    env.put(ApplicationConstants.Environment.USER.name(), request.getUser());
    String[] localDirs = new String[localDirsBase.length];
    // Setup up local dirs to be application specific, and create them.
    for (int i = 0; i < localDirsBase.length; i++) {
        localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(), request.getUser());
        try {
            localFs.mkdirs(new Path(localDirs[i]));
        } catch (IOException e) {
            throw new TezException(e);
        }
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("Dirs are: " + Arrays.toString(localDirs));
    }
    // Setup workingDir. This is otherwise setup as Environment.PWD
    // Used for re-localization, to add the user specified configuration (conf_pb_binary_stream)
    String workingDir = localDirs[0];
    Credentials credentials = new Credentials();
    DataInputBuffer dib = new DataInputBuffer();
    byte[] tokenBytes = request.getCredentialsBinary().toByteArray();
    dib.reset(tokenBytes, tokenBytes.length);
    try {
        credentials.readTokenStorageStream(dib);
    } catch (IOException e) {
        throw new TezException(e);
    }
    Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
    // TODO Unregistering does not happen at the moment, since there's no signals on when an app completes.
    LOG.info("Registering request with the ShuffleHandler for containerId {}", request.getContainerIdString());
    ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser());
    TaskRunnerCallable callable = new TaskRunnerCallable(request, new Configuration(getConfig()), new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs, workingDir, credentials, memoryPerExecutor, sharedExecutor);
    ListenableFuture<ContainerExecutionResult> future = executorService.submit(callable);
    Futures.addCallback(future, new TaskRunnerCallback(request, callable));
}

Example 2 with ContainerExecutionResult

use of org.apache.tez.runtime.task.TezChild.ContainerExecutionResult in project tez by apache.

the class ContainerRunnerImpl method queueContainer.

/**
 * Submit a container which is ready for running.
 * The regular pull mechanism will be used to fetch work from the AM
 * @param request
 * @throws TezException
 */
@Override
public void queueContainer(RunContainerRequestProto request) throws TezException {
    LOG.info("Queuing container for execution: " + request);
    Map<String, String> env = new HashMap<String, String>();
    env.putAll(localEnv);
    env.put(ApplicationConstants.Environment.USER.name(), request.getUser());
    String[] localDirs = new String[localDirsBase.length];
    // Setup up local dirs to be application specific, and create them.
    for (int i = 0; i < localDirsBase.length; i++) {
        localDirs[i] = createAppSpecificLocalDir(localDirsBase[i], request.getApplicationIdString(), request.getUser());
        try {
            localFs.mkdirs(new Path(localDirs[i]));
        } catch (IOException e) {
            throw new TezException(e);
        }
    }
    LOG.info("Dirs for {} are {}", request.getContainerIdString(), Arrays.toString(localDirs));
    // Setup workingDir. This is otherwise setup as Environment.PWD
    // Used for re-localization, to add the user specified configuration (conf_pb_binary_stream)
    String workingDir = localDirs[0];
    Credentials credentials = new Credentials();
    DataInputBuffer dib = new DataInputBuffer();
    byte[] tokenBytes = request.getCredentialsBinary().toByteArray();
    dib.reset(tokenBytes, tokenBytes.length);
    try {
        credentials.readTokenStorageStream(dib);
    } catch (IOException e) {
        throw new TezException(e);
    }
    Token<JobTokenIdentifier> jobToken = TokenCache.getSessionToken(credentials);
    // TODO Unregistering does not happen at the moment, since there's no signals on when an app completes.
    LOG.info("Registering request with the ShuffleHandler for containerId {}", request.getContainerIdString());
    ShuffleHandler.get().registerApplication(request.getApplicationIdString(), jobToken, request.getUser());
    ContainerRunnerCallable callable = new ContainerRunnerCallable(request, new Configuration(getConfig()), new ExecutionContextImpl(localAddress.get().getHostName()), env, localDirs, workingDir, credentials, memoryPerExecutor);
    ListenableFuture<ContainerExecutionResult> future = executorService.submit(callable);
    Futures.addCallback(future, new ContainerRunnerCallback(request, callable));
}