Examples with TaskInfo org.apache.tez.history.parser.datamodel.TaskInfo used on opensource projects

Example 1 with TaskInfo

use of org.apache.tez.history.parser.datamodel.TaskInfo in project tez by apache.

the class ATSFileParser method processTasks.

/**
 * Parse Tasks json
 *
 * @param tasksJson
 * @throws JSONException
 */
private void processTasks(JSONArray tasksJson) throws JSONException {
    // Process Task information
    Preconditions.checkState(tasksJson != null, "Task json can not be null");
    if (tasksJson != null) {
        LOG.debug("Started parsing task");
        for (int i = 0; i < tasksJson.length(); i++) {
            TaskInfo taskInfo = TaskInfo.create(tasksJson.getJSONObject(i));
            taskList.add(taskInfo);
        }
        LOG.debug("Finished parsing task");
    }
}

Example 2 with TaskInfo

use of org.apache.tez.history.parser.datamodel.TaskInfo in project tez by apache.

the class TestHistoryParser method verifyVertex.

private void verifyVertex(VertexInfo vertexInfo, boolean hasFailedTasks) {
    assertTrue(vertexInfo != null);
    if (hasFailedTasks) {
        assertTrue(vertexInfo.getFailedTasksCount() > 0);
    }
    assertTrue(vertexInfo.getStartTimeInterval() > 0);
    assertTrue(vertexInfo.getStartTime() > 0);
    assertTrue(vertexInfo.getFinishTimeInterval() > 0);
    assertTrue(vertexInfo.getStartTimeInterval() < vertexInfo.getFinishTimeInterval());
    assertTrue(vertexInfo.getVertexName() != null);
    if (!hasFailedTasks) {
        assertTrue(vertexInfo.getFinishTime() > 0);
        assertTrue(vertexInfo.getFailedTasks().size() == 0);
        assertTrue(vertexInfo.getSucceededTasksCount() == vertexInfo.getSuccessfulTasks().size());
        assertTrue(vertexInfo.getFailedTasksCount() == 0);
        assertTrue(vertexInfo.getAvgTaskDuration() > 0);
        assertTrue(vertexInfo.getMaxTaskDuration() > 0);
        assertTrue(vertexInfo.getMinTaskDuration() > 0);
        assertTrue(vertexInfo.getTimeTaken() > 0);
        assertTrue(vertexInfo.getStatus().equalsIgnoreCase(VertexState.SUCCEEDED.toString()));
        assertTrue(vertexInfo.getCompletedTasksCount() > 0);
        assertTrue(vertexInfo.getFirstTaskToStart() != null);
        assertTrue(vertexInfo.getSucceededTasksCount() > 0);
        assertTrue(vertexInfo.getTasks().size() > 0);
        assertTrue(vertexInfo.getFinishTime() > vertexInfo.getStartTime());
    }
    for (TaskInfo taskInfo : vertexInfo.getTasks()) {
        if (taskInfo.getStatus().equals(TaskState.SUCCEEDED.toString())) {
            verifyTask(taskInfo, false);
        }
    }
    for (TaskInfo taskInfo : vertexInfo.getFailedTasks()) {
        verifyTask(taskInfo, true);
    }
    assertTrue(vertexInfo.getProcessorClassName() != null);
    assertTrue(vertexInfo.getStatus() != null);
    assertTrue(vertexInfo.getDagInfo() != null);
    assertTrue(vertexInfo.getInitTimeInterval() > 0);
    assertTrue(vertexInfo.getNumTasks() > 0);
}

Example 3 with TaskInfo

use of org.apache.tez.history.parser.datamodel.TaskInfo in project tez by apache.

the class TestHistoryParser method testParserWithFailedJob.

/**
 * Run a failed job and parse the data from ATS
 */
@Test
public void testParserWithFailedJob() throws Exception {
    // Run a job which would fail
    String dagId = runWordCount(WordCount.TokenProcessor.class.getName(), FailProcessor.class.getName(), "WordCount-With-Exception", true);
    // Export the data from ATS
    String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress };
    int result = ATSImportTool.process(args);
    assertTrue(result == 0);
    // Parse ATS data
    DagInfo dagInfo = getDagInfo(dagId);
    // Applicable for ATS dataset
    checkConfig(dagInfo);
    // Verify DAGInfo. Verifies vertex, task, taskAttempts in recursive manner
    verifyDagInfo(dagInfo, true);
    // Dag specific
    VertexInfo summationVertex = dagInfo.getVertex(SUMMATION);
    // 1 task, 4 attempts failed
    assertTrue(summationVertex.getFailedTasks().size() == 1);
    assertTrue(summationVertex.getFailedTasks().get(0).getFailedTaskAttempts().size() == 4);
    assertTrue(summationVertex.getStatus().equals(VertexState.FAILED.toString()));
    assertTrue(dagInfo.getFailedVertices().size() == 1);
    assertTrue(dagInfo.getFailedVertices().get(0).getVertexName().equals(SUMMATION));
    assertTrue(dagInfo.getSuccessfullVertices().size() == 1);
    assertTrue(dagInfo.getSuccessfullVertices().get(0).getVertexName().equals(TOKENIZER));
    assertTrue(dagInfo.getStatus().equals(DAGState.FAILED.toString()));
    verifyCounter(dagInfo.getCounter(DAGCounter.NUM_FAILED_TASKS.toString()), null, 4);
    verifyCounter(dagInfo.getCounter(DAGCounter.NUM_SUCCEEDED_TASKS.toString()), null, 1);
    verifyCounter(dagInfo.getCounter(DAGCounter.TOTAL_LAUNCHED_TASKS.toString()), null, 5);
    verifyCounter(dagInfo.getCounter(TaskCounter.INPUT_RECORDS_PROCESSED.toString()), "TaskCounter_Tokenizer_INPUT_Input", 10);
    verifyCounter(dagInfo.getCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_READ.toString()), "TaskCounter_Tokenizer_OUTPUT_Summation", 0);
    verifyCounter(dagInfo.getCounter(TaskCounter.OUTPUT_RECORDS.toString()), "TaskCounter_Tokenizer_OUTPUT_Summation", // Every line has 2 words. 10 lines x 2 words = 20
    20);
    verifyCounter(dagInfo.getCounter(TaskCounter.SPILLED_RECORDS.toString()), "TaskCounter_Tokenizer_OUTPUT_Summation", // Same as above
    20);
    for (TaskInfo taskInfo : summationVertex.getTasks()) {
        TaskAttemptInfo lastAttempt = null;
        for (TaskAttemptInfo attemptInfo : taskInfo.getTaskAttempts()) {
            if (lastAttempt != null) {
                // failed attempt should be causal TA of next attempt
                assertTrue(lastAttempt.getTaskAttemptId().equals(attemptInfo.getCreationCausalTA()));
                assertTrue(lastAttempt.getTerminationCause() != null);
            }
            lastAttempt = attemptInfo;
        }
    }
    // TODO: Need to check for SUMMATION vertex counters. Since all attempts are failed, counters are not getting populated.
    // TaskCounter.REDUCE_INPUT_RECORDS
    // Verify if the processor exception is given in diagnostics
    assertTrue(dagInfo.getDiagnostics().contains("Failing this processor for some reason"));
}

Example 4 with TaskInfo

use of org.apache.tez.history.parser.datamodel.TaskInfo in project tez by apache.

the class CriticalPathAnalyzer method analyze.

@Override
public void analyze(DagInfo dagInfo) throws TezException {
    // get all attempts in the dag and find the last failed/succeeded attempt.
    // ignore killed attempt to handle kills that happen upon dag completion
    TaskAttemptInfo lastAttempt = null;
    long lastAttemptFinishTime = 0;
    for (VertexInfo vertex : dagInfo.getVertices()) {
        for (TaskInfo task : vertex.getTasks()) {
            for (TaskAttemptInfo attempt : task.getTaskAttempts()) {
                attempts.put(attempt.getTaskAttemptId(), attempt);
                if (attempt.getStatus().equals(succeededState) || attempt.getStatus().equals(failedState)) {
                    if (lastAttemptFinishTime < attempt.getFinishTime()) {
                        lastAttempt = attempt;
                        lastAttemptFinishTime = attempt.getFinishTime();
                    }
                }
            }
        }
    }
    if (lastAttempt == null) {
        LOG.info("Cannot find last attempt to finish in DAG " + dagInfo.getDagId());
        return;
    }
    createCriticalPath(dagInfo, lastAttempt, lastAttemptFinishTime, attempts);
    analyzeCriticalPath(dagInfo);
    if (getConf().getBoolean(DRAW_SVG, true)) {
        saveCriticalPathAsSVG(dagInfo);
    }
}

Example 5 with TaskInfo

use of org.apache.tez.history.parser.datamodel.TaskInfo in project tez by apache.

the class CriticalPathAnalyzer method determineConcurrency.

private void determineConcurrency(DagInfo dag) {
    ArrayList<TimeInfo> timeInfo = Lists.newArrayList();
    for (VertexInfo v : dag.getVertices()) {
        for (TaskInfo t : v.getTasks()) {
            for (TaskAttemptInfo a : t.getTaskAttempts()) {
                if (a.getStartTime() > 0) {
                    timeInfo.add(new TimeInfo(a.getStartTime(), true));
                    timeInfo.add(new TimeInfo(a.getFinishTime(), false));
                }
            }
        }
    }
    Collections.sort(timeInfo);
    int concurrency = 0;
    TimeInfo lastTimeInfo = null;
    for (TimeInfo t : timeInfo) {
        concurrency += (t.start) ? 1 : -1;
        maxConcurrency = (concurrency > maxConcurrency) ? concurrency : maxConcurrency;
        if (lastTimeInfo == null || lastTimeInfo.timestamp < t.timestamp) {
            lastTimeInfo = t;
            lastTimeInfo.count = concurrency;
            concurrencyByTime.add(lastTimeInfo);
        } else {
            // lastTimeInfo.timestamp == t.timestamp
            lastTimeInfo.count = concurrency;
        }
    }
// for (TimeInfo t : concurrencyByTime) {
// System.out.println(t.timestamp + " " + t.count);
// }
}