Examples with TezTaskRunner2 org.apache.tez.runtime.task.TezTaskRunner2 used on opensource projects

Example 1 with TezTaskRunner2

use of org.apache.tez.runtime.task.TezTaskRunner2 in project hive by apache.

the class TaskRunnerCallable method callInternal.

@Override
protected TaskRunner2Result callInternal() throws Exception {
    setMDCFromNDC();
    try {
        isStarted.set(true);
        this.startTime = System.currentTimeMillis();
        threadName = Thread.currentThread().getName();
        this.threadName = Thread.currentThread().getName();
        if (LOG.isDebugEnabled()) {
            LOG.debug("canFinish: " + taskSpec.getTaskAttemptID() + ": " + canFinish());
        }
        // Unregister from the AMReporter, since the task is now running.
        TezTaskAttemptID ta = taskSpec.getTaskAttemptID();
        this.amReporter.unregisterTask(request.getAmHost(), request.getAmPort(), fragmentInfo.getQueryInfo().getQueryIdentifier(), ta);
        synchronized (this) {
            if (!shouldRunTask) {
                LOG.info("Not starting task {} since it was killed earlier", ta);
                return new TaskRunner2Result(EndReason.KILL_REQUESTED, null, null, false);
            }
        }
        // TODO This executor seems unnecessary. Here and TezChild
        executor = new StatsRecordingThreadPool(1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<Runnable>(), new ThreadFactoryBuilder().setDaemon(true).setNameFormat("TezTR-" + threadNameSuffix).build());
        // TODO Consolidate this code with TezChild.
        runtimeWatch.start();
        if (fsTaskUgi == null) {
            fsTaskUgi = UserGroupInformation.createRemoteUser(vertex.getUser());
        }
        fsTaskUgi.addCredentials(credentials);
        Map<String, ByteBuffer> serviceConsumerMetadata = new HashMap<>();
        serviceConsumerMetadata.put(TezConstants.TEZ_SHUFFLE_HANDLER_SERVICE_ID, TezCommonUtils.convertJobTokenToBytes(jobToken));
        Multimap<String, String> startedInputsMap = createStartedInputMap(vertex);
        final UserGroupInformation taskOwner;
        if (!vertex.getIsExternalSubmission()) {
            taskOwner = fragmentInfo.getQueryInfo().getUmbilicalUgi();
        } else {
            // Temporary, till the external interface makes use of a single connection per
            // instance.
            taskOwner = UserGroupInformation.createRemoteUser(vertex.getTokenIdentifier());
            taskOwner.addToken(jobToken);
            final InetSocketAddress address = NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort());
            SecurityUtil.setTokenService(jobToken, address);
        }
        if (LOG.isDebugEnabled()) {
            LOG.debug("taskOwner hashCode:" + taskOwner.hashCode());
        }
        final InetSocketAddress address = NetUtils.createSocketAddrForHost(request.getAmHost(), request.getAmPort());
        umbilical = taskOwner.doAs(new PrivilegedExceptionAction<LlapTaskUmbilicalProtocol>() {

            @Override
            public LlapTaskUmbilicalProtocol run() throws Exception {
                return RPC.getProxy(LlapTaskUmbilicalProtocol.class, LlapTaskUmbilicalProtocol.versionID, address, taskOwner, conf, socketFactory);
            }
        });
        String fragmentId = LlapTezUtils.stripAttemptPrefix(taskSpec.getTaskAttemptID().toString());
        taskReporter = new LlapTaskReporter(completionListener, umbilical, confParams.amHeartbeatIntervalMsMax, confParams.amCounterHeartbeatInterval, confParams.amMaxEventsPerHeartbeat, new AtomicLong(0), request.getContainerIdString(), fragmentId, initialEvent, requestId);
        String attemptId = fragmentInfo.getFragmentIdentifierString();
        IOContextMap.setThreadAttemptId(attemptId);
        try {
            synchronized (this) {
                if (shouldRunTask) {
                    taskRunner = new TezTaskRunner2(conf, fsTaskUgi, fragmentInfo.getLocalDirs(), taskSpec, vertex.getQueryIdentifier().getAppAttemptNumber(), serviceConsumerMetadata, envMap, startedInputsMap, taskReporter, executor, objectRegistry, pid, executionContext, memoryAvailable, false, tezHadoopShim);
                }
            }
            if (taskRunner == null) {
                LOG.info("Not starting task {} since it was killed earlier", taskSpec.getTaskAttemptID());
                return new TaskRunner2Result(EndReason.KILL_REQUESTED, null, null, false);
            }
            try {
                TaskRunner2Result result = taskRunner.run();
                if (result.isContainerShutdownRequested()) {
                    LOG.warn("Unexpected container shutdown requested while running task. Ignoring");
                }
                isCompleted.set(true);
                return result;
            } finally {
                FileSystem.closeAllForUGI(fsTaskUgi);
                fragmentInfo.getQueryInfo().returnUmbilicalUgi(taskOwner);
                LOG.info("ExecutionTime for Container: " + request.getContainerIdString() + "=" + runtimeWatch.stop().elapsed(TimeUnit.MILLISECONDS));
                if (LOG.isDebugEnabled()) {
                    LOG.debug("canFinish post completion: " + taskSpec.getTaskAttemptID() + ": " + canFinish());
                }
            }
        } finally {
            IOContextMap.clearThreadAttempt(attemptId);
        }
    } finally {
        MDC.clear();
    }
}