Examples with ContainerContext org.apache.tez.common.ContainerContext used on opensource projects

Example 1 with ContainerContext

use of org.apache.tez.common.ContainerContext in project tez by apache.

the class TezChild method run.

public ContainerExecutionResult run() throws IOException, InterruptedException, TezException {
    ContainerContext containerContext = new ContainerContext(containerIdString);
    ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, getTaskMaxSleepTime);
    taskReporter = new TaskReporter(umbilical, amHeartbeatInterval, sendCounterInterval, maxEventsToGet, heartbeatCounter, containerIdString);
    UserGroupInformation childUGI = null;
    while (!executor.isTerminated() && !isShutdown.get()) {
        if (taskCount > 0) {
            TezUtilsInternal.updateLoggers("");
        }
        ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);
        boolean error = false;
        ContainerTask containerTask = null;
        try {
            containerTask = getTaskFuture.get();
        } catch (ExecutionException e) {
            error = true;
            Throwable cause = e.getCause();
            LOG.error("Error fetching new work for container {}", containerIdString, cause);
            return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE, cause, "Execution Exception while fetching new work: " + e.getMessage());
        } catch (InterruptedException e) {
            error = true;
            LOG.info("Interrupted while waiting for new work for container {}", containerIdString);
            return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.INTERRUPTED, e, "Interrupted while waiting for new work");
        } finally {
            if (error) {
                shutdown();
            }
        }
        if (containerTask.shouldDie()) {
            LOG.info("ContainerTask returned shouldDie=true for container {}, Exiting", containerIdString);
            shutdown();
            return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null, "Asked to die by the AM");
        } else {
            String loggerAddend = containerTask.getTaskSpec().getTaskAttemptID().toString();
            taskCount++;
            String timeStamp = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(Calendar.getInstance().getTime());
            System.err.println(timeStamp + " Starting to run new task attempt: " + containerTask.getTaskSpec().getTaskAttemptID().toString());
            System.out.println(timeStamp + " Starting to run new task attempt: " + containerTask.getTaskSpec().getTaskAttemptID().toString());
            TezUtilsInternal.setHadoopCallerContext(hadoopShim, containerTask.getTaskSpec().getTaskAttemptID());
            TezUtilsInternal.updateLoggers(loggerAddend);
            FileSystem.clearStatistics();
            childUGI = handleNewTaskCredentials(containerTask, childUGI);
            handleNewTaskLocalResources(containerTask, childUGI);
            cleanupOnTaskChanged(containerTask);
            // Execute the Actual Task
            TezTaskRunner2 taskRunner = new TezTaskRunner2(defaultConf, childUGI, localDirs, containerTask.getTaskSpec(), appAttemptNumber, serviceConsumerMetadata, serviceProviderEnvMap, startedInputsMap, taskReporter, executor, objectRegistry, pid, executionContext, memAvailable, updateSysCounters, hadoopShim, sharedExecutor);
            boolean shouldDie;
            try {
                TaskRunner2Result result = taskRunner.run();
                LOG.info("TaskRunner2Result: {}", result);
                shouldDie = result.isContainerShutdownRequested();
                if (shouldDie) {
                    LOG.info("Got a shouldDie notification via heartbeats for container {}. Shutting down", containerIdString);
                    shutdown();
                    return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null, "Asked to die by the AM");
                }
                if (result.getError() != null) {
                    Throwable e = result.getError();
                    handleError(result.getError());
                    return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.EXECUTION_FAILURE, e, "TaskExecutionFailure: " + e.getMessage());
                }
            } finally {
                FileSystem.closeAllForUGI(childUGI);
            }
        }
    }
    return new ContainerExecutionResult(ContainerExecutionResult.ExitStatus.SUCCESS, null, null);
}

Example 2 with ContainerContext

use of org.apache.tez.common.ContainerContext in project tez by apache.

the class TestContainerExecution method testGetTaskShouldDie.

@Test(timeout = 5000)
public void testGetTaskShouldDie() throws InterruptedException, ExecutionException {
    ListeningExecutorService executor = null;
    try {
        ExecutorService rawExecutor = Executors.newFixedThreadPool(1);
        executor = MoreExecutors.listeningDecorator(rawExecutor);
        ApplicationId appId = ApplicationId.newInstance(10000, 1);
        ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
        @SuppressWarnings("deprecation") ContainerId containerId = ContainerId.newInstance(appAttemptId, 1);
        TaskExecutionTestHelpers.TezTaskUmbilicalForTest umbilical = new TaskExecutionTestHelpers.TezTaskUmbilicalForTest();
        ContainerContext containerContext = new ContainerContext(containerId.toString());
        ContainerReporter containerReporter = new ContainerReporter(umbilical, containerContext, 100);
        ListenableFuture<ContainerTask> getTaskFuture = executor.submit(containerReporter);
        getTaskFuture.get();
        assertEquals(1, umbilical.getTaskInvocations);
    } finally {
        executor.shutdownNow();
    }
}

Example 3 with ContainerContext

use of org.apache.tez.common.ContainerContext in project tez by apache.

the class TestTezTaskCommunicatorManager method testContainerAliveOnGetTask.

@Test(timeout = 5000)
public void testContainerAliveOnGetTask() throws IOException {
    TaskCommunicatorContext context = mock(TaskCommunicatorContext.class);
    Configuration conf = new Configuration(false);
    UserPayload userPayload = TezUtils.createUserPayloadFromConf(conf);
    ApplicationId appId = ApplicationId.newInstance(1000, 1);
    ApplicationAttemptId appAttemptId = ApplicationAttemptId.newInstance(appId, 1);
    ContainerId containerId = createContainerId(appId, 1);
    doReturn(appAttemptId).when(context).getApplicationAttemptId();
    doReturn(userPayload).when(context).getInitialUserPayload();
    doReturn(new Credentials()).when(context).getAMCredentials();
    TezTaskCommunicatorImpl taskComm = new TezTaskCommunicatorImpl(context);
    ContainerContext containerContext = new ContainerContext(containerId.toString());
    taskComm.registerRunningContainer(containerId, "fakehost", 0);
    ContainerTask containerTask = taskComm.getUmbilical().getTask(containerContext);
    assertNull(containerTask);
    verify(context).containerAlive(containerId);
}

Example 4 with ContainerContext

use of org.apache.tez.common.ContainerContext in project tez by apache.

the class TestTaskCommunicatorManager1 method testGetTask.

@Test(timeout = 5000)
public void testGetTask() throws IOException {
    TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl) taskAttemptListener.getTaskCommunicator(0).getTaskCommunicator();
    TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
    ContainerId containerId1 = createContainerId(appId, 1);
    ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
    containerTask = tezUmbilical.getTask(containerContext1);
    assertTrue(containerTask.shouldDie());
    ContainerId containerId2 = createContainerId(appId, 2);
    ContainerContext containerContext2 = new ContainerContext(containerId2.toString());
    taskAttemptListener.registerRunningContainer(containerId2, 0);
    containerTask = tezUmbilical.getTask(containerContext2);
    assertNull(containerTask);
    // Valid task registered
    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId2, 0);
    containerTask = tezUmbilical.getTask(containerContext2);
    assertFalse(containerTask.shouldDie());
    assertEquals(taskSpec, containerTask.getTaskSpec());
    // Task unregistered. Should respond to heartbeats
    taskAttemptListener.unregisterTaskAttempt(taskAttemptID, 0, TaskAttemptEndReason.OTHER, null);
    containerTask = tezUmbilical.getTask(containerContext2);
    assertNull(containerTask);
    // Container unregistered. Should send a shouldDie = true
    taskAttemptListener.unregisterRunningContainer(containerId2, 0, ContainerEndReason.OTHER, null);
    containerTask = tezUmbilical.getTask(containerContext2);
    assertTrue(containerTask.shouldDie());
    ContainerId containerId3 = createContainerId(appId, 3);
    ContainerContext containerContext3 = new ContainerContext(containerId3.toString());
    taskAttemptListener.registerRunningContainer(containerId3, 0);
    // Register task to container3, followed by unregistering container 3 all together
    TaskSpec taskSpec2 = mock(TaskSpec.class);
    TezTaskAttemptID taskAttemptId2 = mock(TezTaskAttemptID.class);
    doReturn(taskAttemptId2).when(taskSpec2).getTaskAttemptID();
    AMContainerTask amContainerTask2 = new AMContainerTask(taskSpec, null, null, false, 0);
    taskAttemptListener.registerTaskAttempt(amContainerTask2, containerId3, 0);
    taskAttemptListener.unregisterRunningContainer(containerId3, 0, ContainerEndReason.OTHER, null);
    containerTask = tezUmbilical.getTask(containerContext3);
    assertTrue(containerTask.shouldDie());
}

Example 5 with ContainerContext

use of org.apache.tez.common.ContainerContext in project tez by apache.

the class TestTaskCommunicatorManager1 method testGetTaskMultiplePulls.

@Test(timeout = 5000)
public void testGetTaskMultiplePulls() throws IOException {
    TezTaskCommunicatorImpl taskCommunicator = (TezTaskCommunicatorImpl) taskAttemptListener.getTaskCommunicator(0).getTaskCommunicator();
    TezTaskUmbilicalProtocol tezUmbilical = taskCommunicator.getUmbilical();
    ContainerId containerId1 = createContainerId(appId, 1);
    ContainerContext containerContext1 = new ContainerContext(containerId1.toString());
    taskAttemptListener.registerRunningContainer(containerId1, 0);
    containerTask = tezUmbilical.getTask(containerContext1);
    assertNull(containerTask);
    // Register task
    taskAttemptListener.registerTaskAttempt(amContainerTask, containerId1, 0);
    containerTask = tezUmbilical.getTask(containerContext1);
    assertFalse(containerTask.shouldDie());
    assertEquals(taskSpec, containerTask.getTaskSpec());
    // Try pulling again - simulates re-use pull
    containerTask = tezUmbilical.getTask(containerContext1);
    assertNull(containerTask);
}