Examples with TaskAttemptInfo org.apache.tez.history.parser.datamodel.TaskAttemptInfo used on opensource projects

Example 1 with TaskAttemptInfo

use of org.apache.tez.history.parser.datamodel.TaskAttemptInfo in project tez by apache.

the class TestHistoryParser method testParserWithFailedJob.

/**
 * Run a failed job and parse the data from ATS
 */
@Test
public void testParserWithFailedJob() throws Exception {
    // Run a job which would fail
    String dagId = runWordCount(WordCount.TokenProcessor.class.getName(), FailProcessor.class.getName(), "WordCount-With-Exception", true);
    // Export the data from ATS
    String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress };
    int result = ATSImportTool.process(args);
    assertTrue(result == 0);
    // Parse ATS data
    DagInfo dagInfo = getDagInfo(dagId);
    // Applicable for ATS dataset
    checkConfig(dagInfo);
    // Verify DAGInfo. Verifies vertex, task, taskAttempts in recursive manner
    verifyDagInfo(dagInfo, true);
    // Dag specific
    VertexInfo summationVertex = dagInfo.getVertex(SUMMATION);
    // 1 task, 4 attempts failed
    assertTrue(summationVertex.getFailedTasks().size() == 1);
    assertTrue(summationVertex.getFailedTasks().get(0).getFailedTaskAttempts().size() == 4);
    assertTrue(summationVertex.getStatus().equals(VertexState.FAILED.toString()));
    assertTrue(dagInfo.getFailedVertices().size() == 1);
    assertTrue(dagInfo.getFailedVertices().get(0).getVertexName().equals(SUMMATION));
    assertTrue(dagInfo.getSuccessfullVertices().size() == 1);
    assertTrue(dagInfo.getSuccessfullVertices().get(0).getVertexName().equals(TOKENIZER));
    assertTrue(dagInfo.getStatus().equals(DAGState.FAILED.toString()));
    verifyCounter(dagInfo.getCounter(DAGCounter.NUM_FAILED_TASKS.toString()), null, 4);
    verifyCounter(dagInfo.getCounter(DAGCounter.NUM_SUCCEEDED_TASKS.toString()), null, 1);
    verifyCounter(dagInfo.getCounter(DAGCounter.TOTAL_LAUNCHED_TASKS.toString()), null, 5);
    verifyCounter(dagInfo.getCounter(TaskCounter.INPUT_RECORDS_PROCESSED.toString()), "TaskCounter_Tokenizer_INPUT_Input", 10);
    verifyCounter(dagInfo.getCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_READ.toString()), "TaskCounter_Tokenizer_OUTPUT_Summation", 0);
    verifyCounter(dagInfo.getCounter(TaskCounter.OUTPUT_RECORDS.toString()), "TaskCounter_Tokenizer_OUTPUT_Summation", // Every line has 2 words. 10 lines x 2 words = 20
    20);
    verifyCounter(dagInfo.getCounter(TaskCounter.SPILLED_RECORDS.toString()), "TaskCounter_Tokenizer_OUTPUT_Summation", // Same as above
    20);
    for (TaskInfo taskInfo : summationVertex.getTasks()) {
        TaskAttemptInfo lastAttempt = null;
        for (TaskAttemptInfo attemptInfo : taskInfo.getTaskAttempts()) {
            if (lastAttempt != null) {
                // failed attempt should be causal TA of next attempt
                assertTrue(lastAttempt.getTaskAttemptId().equals(attemptInfo.getCreationCausalTA()));
                assertTrue(lastAttempt.getTerminationCause() != null);
            }
            lastAttempt = attemptInfo;
        }
    }
    // TODO: Need to check for SUMMATION vertex counters. Since all attempts are failed, counters are not getting populated.
    // TaskCounter.REDUCE_INPUT_RECORDS
    // Verify if the processor exception is given in diagnostics
    assertTrue(dagInfo.getDiagnostics().contains("Failing this processor for some reason"));
}

Example 2 with TaskAttemptInfo

use of org.apache.tez.history.parser.datamodel.TaskAttemptInfo in project tez by apache.

the class TestHistoryParser method verifyTask.

private void verifyTask(TaskInfo taskInfo, boolean hasFailedAttempts) {
    assertTrue(taskInfo != null);
    assertTrue(taskInfo.getStatus() != null);
    assertTrue(taskInfo.getStartTimeInterval() > 0);
    // Not testing for killed attempts. So if there are no failures, it should succeed
    if (!hasFailedAttempts) {
        assertTrue(taskInfo.getStatus().equals(TaskState.SUCCEEDED.toString()));
        assertTrue(taskInfo.getFinishTimeInterval() > 0 && taskInfo.getFinishTime() > taskInfo.getFinishTimeInterval());
        assertTrue(taskInfo.getStartTimeInterval() > 0 && taskInfo.getStartTime() > taskInfo.getStartTimeInterval());
        assertTrue(taskInfo.getSuccessfulAttemptId() != null);
        assertTrue(taskInfo.getSuccessfulTaskAttempt() != null);
        assertTrue(taskInfo.getFinishTime() > taskInfo.getStartTime());
    }
    assertTrue(taskInfo.getTaskId() != null);
    for (TaskAttemptInfo attemptInfo : taskInfo.getTaskAttempts()) {
        verifyTaskAttemptInfo(attemptInfo);
    }
}

Example 3 with TaskAttemptInfo

use of org.apache.tez.history.parser.datamodel.TaskAttemptInfo in project tez by apache.

the class ContainerReuseAnalyzer method analyze.

@Override
public void analyze(DagInfo dagInfo) throws TezException {
    for (VertexInfo vertexInfo : dagInfo.getVertices()) {
        Multimap<Container, TaskAttemptInfo> containers = vertexInfo.getContainersMapping();
        for (Container container : containers.keySet()) {
            List<String> record = Lists.newLinkedList();
            record.add(vertexInfo.getVertexName());
            record.add(vertexInfo.getTaskAttempts().size() + "");
            record.add(container.getHost());
            record.add(container.getId());
            record.add(Integer.toString(containers.get(container).size()));
            csvResult.addRecord(record.toArray(new String[record.size()]));
        }
    }
}

Example 4 with TaskAttemptInfo

use of org.apache.tez.history.parser.datamodel.TaskAttemptInfo in project tez by apache.

the class SlowNodeAnalyzer method getAvgTaskExecutionTime.

private float getAvgTaskExecutionTime(Iterable<TaskAttemptInfo> taskAttemptInfos) {
    long totalTime = 0;
    int size = 0;
    for (TaskAttemptInfo attemptInfo : taskAttemptInfos) {
        totalTime += attemptInfo.getTimeTaken();
        size++;
    }
    return (size > 0) ? (totalTime * 1.0f / size) : 0;
}

Example 5 with TaskAttemptInfo

use of org.apache.tez.history.parser.datamodel.TaskAttemptInfo in project tez by apache.

the class SlowNodeAnalyzer method getAvgCounter.

private float getAvgCounter(Collection<TaskAttemptInfo> taskAttemptInfos, String counterGroupName, String counterName) {
    long total = 0;
    int taskCount = 0;
    for (TaskAttemptInfo attemptInfo : taskAttemptInfos) {
        TezCounters tezCounters = attemptInfo.getTezCounters();
        TezCounter counter = tezCounters.findCounter(counterGroupName, counterName);
        if (counter != null) {
            total += counter.getValue();
            taskCount++;
        } else {
            LOG.info("Could not find counterGroupName=" + counterGroupName + ", counter=" + counterName + " in " + attemptInfo);
        }
    }
    return (taskCount > 0) ? (total * 1.0f / taskCount) : 0;
}