Examples with ContainerTask org.apache.tez.common.ContainerTask used on opensource projects

Example 1 with ContainerTask

use of org.apache.tez.common.ContainerTask in project tez by apache.

the class TezTaskCommunicatorImpl method getContainerTask.

private ContainerTask getContainerTask(ContainerId containerId) throws IOException {
    ContainerInfo containerInfo = registeredContainers.get(containerId);
    ContainerTask task;
    if (containerInfo == null) {
        if (getContext().isKnownContainer(containerId)) {
            LOG.info("Container with id: " + containerId + " is valid, but no longer registered, and will be killed");
        } else {
            LOG.info("Container with id: " + containerId + " is invalid and will be killed");
        }
        task = TASK_FOR_INVALID_JVM;
    } else {
        synchronized (containerInfo) {
            getContext().containerAlive(containerId);
            if (containerInfo.taskSpec != null) {
                if (!containerInfo.taskPulled) {
                    containerInfo.taskPulled = true;
                    task = constructContainerTask(containerInfo);
                } else {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Task " + containerInfo.taskSpec.getTaskAttemptID() + " already sent to container: " + containerId);
                    }
                    task = null;
                }
            } else {
                task = null;
                if (LOG.isDebugEnabled()) {
                    LOG.debug("No task assigned yet for running container: " + containerId);
                }
            }
        }
    }
    return task;
}

Example 2 with ContainerTask

use of org.apache.tez.common.ContainerTask in project tez by apache.

the class TezChild method run.

public ContainerExecutionResult run() throws IOException, InterruptedException, TezException {
    ContainerContext containerContext = new ContainerContext(containerIdString);
    ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, getTaskMaxSleepTime);
    taskReporter = new TaskReporter(umbilical, amHeartbeatInterval, sendCounterInterval, maxEventsToGet, heartbeatCounter, containerIdString);
    UserGroupInformation childUGI = null;
    while (!executor.isTerminated() && !isShutdown.get()) {
        if (taskCount > 0) {
            TezUtilsInternal.updateLoggers("");
        }
        ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);
        boolean error = false;
        ContainerTask containerTask = null;
        try {
            containerTask = getTaskFuture.get();
        } catch (ExecutionException e) {
            error = true;
            Throwable cause = e.getCause();
            LOG.error("Error fetching new work for container {}", containerIdString, cause);
            return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE, cause, "Execution Exception while fetching new work: " + e.getMessage());
        } catch (InterruptedException e) {
            error = true;
            LOG.info("Interrupted while waiting for new work for container {}", containerIdString);
            return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.INTERRUPTED, e, "Interrupted while waiting for new work");
        } finally {
            if (error) {
                shutdown();
            }
        }
        if (containerTask.shouldDie()) {
            LOG.info("ContainerTask returned shouldDie=true for container {}, Exiting", containerIdString);
            shutdown();
            return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null, "Asked to die by the AM");
        } else {
            String loggerAddend = containerTask.getTaskSpec().getTaskAttemptID().toString();
            taskCount++;
            String timeStamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime());
            System.err.println(timeStamp + " Starting to run new task attempt: " + containerTask.getTaskSpec().getTaskAttemptID().toString());
            System.out.println(timeStamp + " Starting to run new task attempt: " + containerTask.getTaskSpec().getTaskAttemptID().toString());
            TezUtilsInternal.setHadoopCallerContext(hadoopShim, containerTask.getTaskSpec().getTaskAttemptID());
            TezUtilsInternal.updateLoggers(loggerAddend);
            FileSystem.clearStatistics();
            childUGI = handleNewTaskCredentials(containerTask, childUGI);
            handleNewTaskLocalResources(containerTask, childUGI);
            cleanupOnTaskChanged(containerTask);
            // Execute the Actual Task
            TezTaskRunner2 taskRunner = new TezTaskRunner2(defaultConf, childUGI, localDirs, containerTask.getTaskSpec(), appAttemptNumber, serviceConsumerMetadata, serviceProviderEnvMap, startedInputsMap, taskReporter, executor, objectRegistry, pid, executionContext, memAvailable, updateSysCounters, hadoopShim, sharedExecutor);
            boolean shouldDie;
            try {
                TaskRunner2Result result = taskRunner.run();
                LOG.info("TaskRunner2Result: {}", result);
                shouldDie = result.isContainerShutdownRequested();
                if (shouldDie) {
                    LOG.info("Got a shouldDie notification via heartbeats for container {}. Shutting down", containerIdString);
                    shutdown();
                    return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null, "Asked to die by the AM");
                }
                if (result.getError() != null) {
                    Throwable e = result.getError();
                    handleError(result.getError());
                    return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE, e, "TaskExecutionFailure: " + e.getMessage());
                }
            } finally {
                FileSystem.closeAllForUGI(childUGI);
            }
        }
    }
    return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null, null);
}

Example 3 with ContainerTask

use of org.apache.tez.common.ContainerTask in project tez by apache.

the class TestContainerExecution method testGetTaskShouldDie.

@Test(timeout = 5000)
public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        @SuppressWarnings("deprecation") ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        ContainerContext containerContext = new ContainerContext(containerId.toString());
        ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
        ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);
        getTaskFuture.get();
        assertEquals(1, umbilical.getTaskInvocations);
    } finally {
        executor.shutdownNow();
    }
}

Example 4 with ContainerTask

use of org.apache.tez.common.ContainerTask in project tez by apache.

the class TestTezTaskCommunicatorManager method testContainerAliveOnGetTask.

@Test(timeout = 5000)
public void testContainerAliveOnGetTask() throws IOException {
    TaskCommunicatorContext context = mock(TaskCommunicatorContext.class);
    Configuration conf = new Configuration(false);
    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId = createContainerId(appId, 1);
    doReturn(appAttemptId).when(context).getApplicationAttemptId();
    doReturn(userPayload).when(context).getInitialUserPayload();
    doReturn(new Credentials()).when(context).getAMCredentials();
    TezTaskCommunicatorImpl taskComm = new TezTaskCommunicatorImpl(context);
    ContainerContext containerContext = new ContainerContext(containerId.toString());
    taskComm.registerRunningContainer(containerId, "fakehost", 0);
    ContainerTask containerTask = taskComm.getUmbilical().getTask(containerContext);
    assertNull(containerTask);
    verify(context).containerAlive(containerId);
}

Example 5 with ContainerTask

use of org.apache.tez.common.ContainerTask in project tez by apache.

the class ContainerReporter method callInternal.

@Override
protected ContainerTask callInternal() throws Exception {
    ContainerTask containerTask = null;
    LOG.info("Attempting to fetch new task for container {}", containerContext.getContainerIdentifier());
    containerTask = umbilical.getTask(containerContext);
    long getTaskPollStartTime = System.currentTimeMillis();
    nextGetTaskPrintTime = getTaskPollStartTime + LOG_INTERVAL;
    for (int idle = 1; containerTask == null; idle++) {
        long sleepTimeMilliSecs = Math.min(idle * 10, getTaskMaxSleepTime);
        maybeLogSleepMessage(sleepTimeMilliSecs);
        TimeUnit.MILLISECONDS.sleep(sleepTimeMilliSecs);
        containerTask = umbilical.getTask(containerContext);
    }
    LOG.info("Got TaskUpdate for containerId= " + containerContext.getContainerIdentifier() + ": " + (System.currentTimeMillis() - getTaskPollStartTime) + " ms after starting to poll." + " TaskInfo: shouldDie: " + containerTask.shouldDie() + (containerTask.shouldDie() == true ? "" : ", currentTaskAttemptId: " + containerTask.getTaskSpec().getTaskAttemptID()));
    return containerTask;
}