Examples with Task org.apache.tez.dag.app.dag.Task used on opensource projects

Example 6 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class Edge method sendTezEventToSourceTasks.

public void sendTezEventToSourceTasks(TezEvent tezEvent) throws AMUserCodeException {
    Preconditions.checkState(edgeManager != null, "Edge Manager must be initialized by this time");
    if (!bufferEvents.get()) {
        switch(tezEvent.getEventType()) {
            case INPUT_READ_ERROR_EVENT:
                InputReadErrorEvent event = (InputReadErrorEvent) tezEvent.getEvent();
                TezTaskAttemptID destAttemptId = tezEvent.getSourceInfo().getTaskAttemptID();
                int destTaskIndex = destAttemptId.getTaskID().getId();
                int srcTaskIndex;
                int numConsumers;
                try {
                    if (onDemandRouting) {
                        srcTaskIndex = ((EdgeManagerPluginOnDemand) edgeManager).routeInputErrorEventToSource(destTaskIndex, event.getIndex());
                    } else {
                        srcTaskIndex = edgeManager.routeInputErrorEventToSource(event, destTaskIndex, event.getIndex());
                    }
                    Preconditions.checkArgument(srcTaskIndex >= 0, "SourceTaskIndex should not be negative," + "srcTaskIndex=" + srcTaskIndex);
                    numConsumers = edgeManager.getNumDestinationConsumerTasks(srcTaskIndex);
                    Preconditions.checkArgument(numConsumers > 0, "ConsumerTaskNum must be positive," + "numConsumers=" + numConsumers);
                } catch (Exception e) {
                    throw new AMUserCodeException(Source.EdgeManager, "Fail to sendTezEventToSourceTasks, " + "TezEvent:" + tezEvent.getEvent() + "sourceInfo:" + tezEvent.getSourceInfo() + "destinationInfo:" + tezEvent.getDestinationInfo() + ", " + getEdgeInfo(), e);
                }
                Task srcTask = sourceVertex.getTask(srcTaskIndex);
                if (srcTask == null) {
                    throw new TezUncheckedException("Unexpected null task." + " sourceVertex=" + sourceVertex.getLogIdentifier() + " srcIndex = " + srcTaskIndex + " destAttemptId=" + destAttemptId + " destIndex=" + destTaskIndex + " edgeManager=" + edgeManager.getClass().getName());
                }
                TezTaskID srcTaskId = srcTask.getTaskId();
                int taskAttemptIndex = event.getVersion();
                TezTaskAttemptID srcTaskAttemptId = TezTaskAttemptID.getInstance(srcTaskId, taskAttemptIndex);
                sendEvent(new TaskAttemptEventOutputFailed(srcTaskAttemptId, tezEvent, numConsumers));
                break;
            default:
                throw new TezUncheckedException("Unhandled tez event type: " + tezEvent.getEventType());
        }
    } else {
        sourceEventBuffer.add(tezEvent);
    }
}

Example 7 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class TaskCommunicatorManager method canCommit.

/**
 * Child checking whether it can commit.
 * <p/>
 * <br/>
 * Repeatedly polls the ApplicationMaster whether it
 * {@link Task#canCommit(TezTaskAttemptID)} This is * a legacy from the
 * centralized commit protocol handling by the JobTracker.
 */
// @Override
public boolean canCommit(TezTaskAttemptID taskAttemptId) throws IOException {
    // An attempt is asking if it can commit its output. This can be decided
    // only by the task which is managing the multiple attempts. So redirect the
    // request there.
    taskHeartbeatHandler.progressing(taskAttemptId);
    pingContainerHeartbeatHandler(taskAttemptId);
    DAG job = context.getCurrentDAG();
    Task task = job.getVertex(taskAttemptId.getTaskID().getVertexID()).getTask(taskAttemptId.getTaskID());
    return task.canCommit(taskAttemptId);
}

Example 8 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class LegacySpeculator method statusUpdate.

/**
 * Absorbs one TaskAttemptStatus
 *
 * @param reportedStatus the status report that we got from a task attempt
 *        that we want to fold into the speculation data for this job
 * @param timestamp the time this status corresponds to.  This matters
 *        because statuses contain progress.
 */
private void statusUpdate(TezTaskAttemptID attemptID, TaskAttemptState reportedState, long timestamp) {
    TezTaskID taskID = attemptID.getTaskID();
    Task task = vertex.getTask(taskID);
    Preconditions.checkState(task != null, "Null task for attempt: " + attemptID);
    estimator.updateAttempt(attemptID, reportedState, timestamp);
    // if (stateString.equals(TaskAttemptState.RUNNING.name())) {
    if (reportedState == TaskAttemptState.RUNNING) {
        runningTasks.putIfAbsent(taskID, Boolean.TRUE);
    } else {
        runningTasks.remove(taskID, Boolean.TRUE);
        // if (!stateString.equals(TaskAttemptState.STARTING.name())) {
        if (reportedState == TaskAttemptState.STARTING) {
            runningTaskAttemptStatistics.remove(attemptID);
        }
    }
}

Example 9 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class LegacyTaskRuntimeEstimator method updateAttempt.

@Override
public void updateAttempt(TezTaskAttemptID attemptID, TaskAttemptState state, long timestamp) {
    super.updateAttempt(attemptID, state, timestamp);
    Task task = vertex.getTask(attemptID.getTaskID());
    if (task == null) {
        return;
    }
    TaskAttempt taskAttempt = task.getAttempt(attemptID);
    if (taskAttempt == null) {
        return;
    }
    float progress = taskAttempt.getProgress();
    Long boxedStart = startTimes.get(attemptID);
    long start = boxedStart == null ? Long.MIN_VALUE : boxedStart;
    // 
    if (taskAttempt.getState() == TaskAttemptState.RUNNING) {
        // See if this task is already in the registry
        AtomicLong estimateContainer = attemptRuntimeEstimates.get(taskAttempt);
        AtomicLong estimateVarianceContainer = attemptRuntimeEstimateVariances.get(taskAttempt);
        if (estimateContainer == null) {
            if (attemptRuntimeEstimates.get(taskAttempt) == null) {
                attemptRuntimeEstimates.put(taskAttempt, new AtomicLong());
                estimateContainer = attemptRuntimeEstimates.get(taskAttempt);
            }
        }
        if (estimateVarianceContainer == null) {
            attemptRuntimeEstimateVariances.putIfAbsent(taskAttempt, new AtomicLong());
            estimateVarianceContainer = attemptRuntimeEstimateVariances.get(taskAttempt);
        }
        long estimate = -1;
        long varianceEstimate = -1;
        // speculative task attempt if two are already running for this task
        if (start > 0 && timestamp > start) {
            estimate = (long) ((timestamp - start) / Math.max(0.0001, progress));
            varianceEstimate = (long) (estimate * progress / 10);
        }
        if (estimateContainer != null) {
            estimateContainer.set(estimate);
        }
        if (estimateVarianceContainer != null) {
            estimateVarianceContainer.set(varianceEstimate);
        }
    }
}

Example 10 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class TestVertexImpl method testVertexScheduleSendEvent.

@Test(timeout = 5000)
public void testVertexScheduleSendEvent() throws Exception {
    VertexImpl v3 = vertices.get("vertex3");
    v3.vertexReconfigurationPlanned();
    initAllVertices(VertexState.INITED);
    Assert.assertEquals(2, v3.getTotalTasks());
    Map<TezTaskID, Task> tasks = v3.getTasks();
    Assert.assertEquals(2, tasks.size());
    VertexImpl v1 = vertices.get("vertex1");
    startVertex(vertices.get("vertex2"));
    startVertex(v1);
    v3.reconfigureVertex(10, null, null);
    checkTasks(v3, 10);
    taskEventDispatcher.events.clear();
    TaskLocationHint mockLocation = mock(TaskLocationHint.class);
    v3.scheduleTasks(Collections.singletonList(ScheduleTaskRequest.create(0, mockLocation)));
    dispatcher.await();
    Assert.assertEquals(1, taskEventDispatcher.events.size());
    TaskEventScheduleTask event = (TaskEventScheduleTask) taskEventDispatcher.events.get(0);
    Assert.assertEquals(mockLocation, event.getTaskLocationHint());
    Assert.assertNotNull(event.getBaseTaskSpec());
    Assert.assertEquals("foobar", event.getBaseTaskSpec().getTaskConf().get("abc"));
}