Examples with Task org.apache.tez.dag.app.dag.Task used on opensource projects

Example 36 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class AMWebController method getTasksInfo.

/**
 * Renders the response JSON for tasksInfo API
 * The JSON will have an array of task objects under the key tasks.
 */
public void getTasksInfo() {
    if (!setupResponse()) {
        return;
    }
    DAG dag = checkAndGetDAGFromRequest();
    if (dag == null) {
        return;
    }
    int limit = MAX_QUERIED;
    try {
        limit = getQueryParamInt(WebUIService.LIMIT);
    } catch (NumberFormatException e) {
    // Ignore
    }
    List<Task> tasks = getRequestedTasks(dag, limit);
    if (tasks == null) {
        return;
    }
    Map<String, Set<String>> counterNames = getCounterListFromRequest();
    ArrayList<Map<String, Object>> tasksInfo = new ArrayList<Map<String, Object>>();
    for (Task t : tasks) {
        Map<String, Object> taskInfo = new HashMap<String, Object>();
        taskInfo.put("id", t.getTaskId().toString());
        taskInfo.put("progress", Float.toString(t.getProgress()));
        taskInfo.put("status", t.getState().toString());
        try {
            TezCounters counters = t.getCounters();
            Map<String, Map<String, Long>> counterMap = constructCounterMapInfo(counters, counterNames);
            if (counterMap != null && !counterMap.isEmpty()) {
                taskInfo.put("counters", counterMap);
            }
        } catch (LimitExceededException e) {
        // Ignore
        // TODO: add an error message instead for counter key
        }
        tasksInfo.add(taskInfo);
    }
    renderJSON(ImmutableMap.of("tasks", tasksInfo));
}

Example 37 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class VertexImpl method tryEnactKill.

/**
 * Set the terminationCause and send a kill-message to all tasks.
 * The task-kill messages are only sent once.
 */
void tryEnactKill(VertexTerminationCause trigger, TaskTerminationCause taskterminationCause) {
    // In most cases the dag is shutting down due to some error
    TaskAttemptTerminationCause errCause = TaskAttemptTerminationCause.TERMINATED_AT_SHUTDOWN;
    if (taskterminationCause == TaskTerminationCause.DAG_KILL) {
        errCause = TaskAttemptTerminationCause.TERMINATED_BY_CLIENT;
    }
    if (trySetTerminationCause(trigger)) {
        String msg = "Killing tasks in vertex: " + logIdentifier + " due to trigger: " + trigger;
        LOG.info(msg);
        for (Task task : tasks.values()) {
            // attempt was terminated because the vertex is shutting down
            eventHandler.handle(new TaskEventTermination(task.getTaskId(), errCause, msg));
        }
    }
}

Example 38 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class VertexImpl method constructFinalFullcounters.

@Private
public void constructFinalFullcounters() {
    this.fullCounters = new TezCounters();
    this.fullCounters.incrAllCounters(counters);
    this.vertexStats = new VertexStats();
    for (Task t : this.tasks.values()) {
        vertexStats.updateStats(t.getReport());
        TezCounters counters = t.getCounters();
        this.fullCounters.incrAllCounters(counters);
    }
}

Example 39 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class LegacySpeculator method maybeScheduleASpeculation.

private int maybeScheduleASpeculation() {
    int successes = 0;
    long now = clock.getTime();
    int numberSpeculationsAlready = 0;
    int numberRunningTasks = 0;
    Map<TezTaskID, Task> tasks = vertex.getTasks();
    int numberAllowedSpeculativeTasks = (int) Math.max(MINIMUM_ALLOWED_SPECULATIVE_TASKS, PROPORTION_TOTAL_TASKS_SPECULATABLE * tasks.size());
    TezTaskID bestTaskID = null;
    long bestSpeculationValue = -1L;
    boolean shouldUseTimeout = (tasks.size() <= VERTEX_SIZE_THRESHOLD_FOR_TIMEOUT_SPECULATION) && (taskTimeout >= 0);
    // TODO track the tasks that are potentially worth looking at
    for (Map.Entry<TezTaskID, Task> taskEntry : tasks.entrySet()) {
        long mySpeculationValue = speculationValue(taskEntry.getValue(), now, shouldUseTimeout);
        if (mySpeculationValue == ALREADY_SPECULATING) {
            ++numberSpeculationsAlready;
        }
        if (mySpeculationValue != NOT_RUNNING) {
            ++numberRunningTasks;
        }
        if (mySpeculationValue > bestSpeculationValue) {
            bestTaskID = taskEntry.getKey();
            bestSpeculationValue = mySpeculationValue;
        }
    }
    numberAllowedSpeculativeTasks = (int) Math.max(numberAllowedSpeculativeTasks, PROPORTION_RUNNING_TASKS_SPECULATABLE * numberRunningTasks);
    // If we found a speculation target, fire it off
    if (bestTaskID != null && numberAllowedSpeculativeTasks > numberSpeculationsAlready) {
        addSpeculativeAttempt(bestTaskID);
        ++successes;
    }
    return successes;
}

Example 40 with Task

use of org.apache.tez.dag.app.dag.Task in project tez by apache.

the class LegacyTaskRuntimeEstimator method storedPerAttemptValue.

private long storedPerAttemptValue(Map<TaskAttempt, AtomicLong> data, TezTaskAttemptID attemptID) {
    Task task = vertex.getTask(attemptID.getTaskID());
    if (task == null) {
        return -1L;
    }
    TaskAttempt taskAttempt = task.getAttempt(attemptID);
    if (taskAttempt == null) {
        return -1L;
    }
    AtomicLong estimate = data.get(taskAttempt);
    return estimate == null ? -1L : estimate.get();
}