Examples with DagInfo org.apache.tez.history.parser.datamodel.DagInfo used on opensource projects

Example 1 with DagInfo

use of org.apache.tez.history.parser.datamodel.DagInfo in project tez by apache.

the class TestHistoryParser method testParserWithFailedJob.

/**
 * Run a failed job and parse the data from ATS
 */
@Test
public void testParserWithFailedJob() throws Exception {
    // Run a job which would fail
    String dagId = runWordCount(WordCount.TokenProcessor.class.getName(), FailProcessor.class.getName(), "WordCount-With-Exception", true);
    // Export the data from ATS
    String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress };
    int result = ATSImportTool.process(args);
    assertTrue(result == 0);
    // Parse ATS data
    DagInfo dagInfo = getDagInfo(dagId);
    // Applicable for ATS dataset
    checkConfig(dagInfo);
    // Verify DAGInfo. Verifies vertex, task, taskAttempts in recursive manner
    verifyDagInfo(dagInfo, true);
    // Dag specific
    VertexInfo summationVertex = dagInfo.getVertex(SUMMATION);
    // 1 task, 4 attempts failed
    assertTrue(summationVertex.getFailedTasks().size() == 1);
    assertTrue(summationVertex.getFailedTasks().get(0).getFailedTaskAttempts().size() == 4);
    assertTrue(summationVertex.getStatus().equals(VertexState.FAILED.toString()));
    assertTrue(dagInfo.getFailedVertices().size() == 1);
    assertTrue(dagInfo.getFailedVertices().get(0).getVertexName().equals(SUMMATION));
    assertTrue(dagInfo.getSuccessfullVertices().size() == 1);
    assertTrue(dagInfo.getSuccessfullVertices().get(0).getVertexName().equals(TOKENIZER));
    assertTrue(dagInfo.getStatus().equals(DAGState.FAILED.toString()));
    verifyCounter(dagInfo.getCounter(DAGCounter.NUM_FAILED_TASKS.toString()), null, 4);
    verifyCounter(dagInfo.getCounter(DAGCounter.NUM_SUCCEEDED_TASKS.toString()), null, 1);
    verifyCounter(dagInfo.getCounter(DAGCounter.TOTAL_LAUNCHED_TASKS.toString()), null, 5);
    verifyCounter(dagInfo.getCounter(TaskCounter.INPUT_RECORDS_PROCESSED.toString()), "TaskCounter_Tokenizer_INPUT_Input", 10);
    verifyCounter(dagInfo.getCounter(TaskCounter.ADDITIONAL_SPILLS_BYTES_READ.toString()), "TaskCounter_Tokenizer_OUTPUT_Summation", 0);
    verifyCounter(dagInfo.getCounter(TaskCounter.OUTPUT_RECORDS.toString()), "TaskCounter_Tokenizer_OUTPUT_Summation", // Every line has 2 words. 10 lines x 2 words = 20
    20);
    verifyCounter(dagInfo.getCounter(TaskCounter.SPILLED_RECORDS.toString()), "TaskCounter_Tokenizer_OUTPUT_Summation", // Same as above
    20);
    for (TaskInfo taskInfo : summationVertex.getTasks()) {
        TaskAttemptInfo lastAttempt = null;
        for (TaskAttemptInfo attemptInfo : taskInfo.getTaskAttempts()) {
            if (lastAttempt != null) {
                // failed attempt should be causal TA of next attempt
                assertTrue(lastAttempt.getTaskAttemptId().equals(attemptInfo.getCreationCausalTA()));
                assertTrue(lastAttempt.getTerminationCause() != null);
            }
            lastAttempt = attemptInfo;
        }
    }
    // TODO: Need to check for SUMMATION vertex counters. Since all attempts are failed, counters are not getting populated.
    // TaskCounter.REDUCE_INPUT_RECORDS
    // Verify if the processor exception is given in diagnostics
    assertTrue(dagInfo.getDiagnostics().contains("Failing this processor for some reason"));
}

Example 2 with DagInfo

use of org.apache.tez.history.parser.datamodel.DagInfo in project tez by apache.

the class TestAnalyzer method verify.

private void verify(ApplicationId appId, int dagNum, List<StepCheck[]> steps) throws Exception {
    String dagId = TezDAGID.getInstance(appId, dagNum).toString();
    DagInfo dagInfo = getDagInfo(dagId);
    verifyCriticalPath(dagInfo, steps);
}

Example 3 with DagInfo

use of org.apache.tez.history.parser.datamodel.DagInfo in project tez by apache.

the class TezAnalyzerBase method run.

@Override
public int run(String[] args) throws Exception {
    // Parse downloaded contents
    CommandLine cmdLine = null;
    try {
        cmdLine = new GnuParser().parse(buildOptions(), args);
    } catch (ParseException e) {
        System.err.println("Invalid options on command line");
        printUsage();
        return -1;
    }
    saveResults = cmdLine.hasOption(SAVE_RESULTS);
    if (cmdLine.hasOption(HELP)) {
        printUsage();
        return 0;
    }
    outputDir = cmdLine.getOptionValue(OUTPUT_DIR);
    if (outputDir == null) {
        outputDir = System.getProperty("user.dir");
    }
    File file = null;
    if (cmdLine.hasOption(EVENT_FILE_NAME)) {
        file = new File(cmdLine.getOptionValue(EVENT_FILE_NAME));
    }
    String dagId = cmdLine.getOptionValue(DAG_ID);
    DagInfo dagInfo = null;
    if (file == null) {
        if (cmdLine.hasOption(FROM_SIMPLE_HISTORY)) {
            System.err.println("Event file name must be specified when using simple history");
            printUsage();
            return -2;
        }
        // using ATS - try to download directly
        String[] importArgs = { "--dagId=" + dagId, "--downloadDir=" + outputDir };
        int result = ATSImportTool.process(importArgs);
        if (result != 0) {
            System.err.println("Error downloading data from ATS");
            return -3;
        }
        // Parse ATS data and verify results
        // Parse downloaded contents
        file = new File(outputDir + Path.SEPARATOR + dagId + ".zip");
    }
    Preconditions.checkState(file != null);
    if (!cmdLine.hasOption(FROM_SIMPLE_HISTORY)) {
        ATSFileParser parser = new ATSFileParser(file);
        dagInfo = parser.getDAGData(dagId);
    } else {
        SimpleHistoryParser parser = new SimpleHistoryParser(file);
        dagInfo = parser.getDAGData(dagId);
    }
    Preconditions.checkState(dagInfo.getDagId().equals(dagId));
    analyze(dagInfo);
    Result result = getResult();
    if (saveResults && (result instanceof CSVResult)) {
        String fileName = outputDir + File.separator + this.getClass().getName() + "_" + dagInfo.getDagId() + ".csv";
        ((CSVResult) result).dumpToFile(fileName);
        LOG.info("Saved results in " + fileName);
    }
    return 0;
}

Example 4 with DagInfo

use of org.apache.tez.history.parser.datamodel.DagInfo in project tez by apache.

the class TestAnalyzer method getDagInfo.

private DagInfo getDagInfo(String dagId) throws Exception {
    // sleep for a bit to let ATS events be sent from AM
    DagInfo dagInfo = null;
    if (usingATS) {
        // Export the data from ATS
        String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress };
        int result = ATSImportTool.process(args);
        assertTrue(result == 0);
        // Parse ATS data and verify results
        // Parse downloaded contents
        File downloadedFile = new File(DOWNLOAD_DIR + Path.SEPARATOR + dagId + ".zip");
        ATSFileParser parser = new ATSFileParser(downloadedFile);
        dagInfo = parser.getDAGData(dagId);
        assertTrue(dagInfo.getDagId().equals(dagId));
    } else {
        if (!downloadedSimpleHistoryFile) {
            downloadedSimpleHistoryFile = true;
            TezDAGID tezDAGID = TezDAGID.fromString(dagId);
            ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(tezDAGID.getApplicationId(), 1);
            Path historyPath = new Path(miniTezCluster.getConfig().get("fs.defaultFS") + SIMPLE_HISTORY_DIR + HISTORY_TXT + "." + applicationAttemptId);
            FileSystem fs = historyPath.getFileSystem(miniTezCluster.getConfig());
            Path localPath = new Path(DOWNLOAD_DIR, HISTORY_TXT);
            fs.copyToLocalFile(historyPath, localPath);
        }
        // Now parse via SimpleHistory
        File localFile = new File(DOWNLOAD_DIR, HISTORY_TXT);
        SimpleHistoryParser parser = new SimpleHistoryParser(localFile);
        dagInfo = parser.getDAGData(dagId);
        assertTrue(dagInfo.getDagId().equals(dagId));
    }
    return dagInfo;
}

Example 5 with DagInfo

use of org.apache.tez.history.parser.datamodel.DagInfo in project tez by apache.

the class SlowestVertexAnalyzer method getTaskRuntime.

private long getTaskRuntime(VertexInfo vertexInfo) {
    TaskInfo firstTaskToStart = vertexInfo.getFirstTaskToStart();
    TaskInfo lastTaskToFinish = vertexInfo.getLastTaskToFinish();
    DagInfo dagInfo = vertexInfo.getDagInfo();
    long totalTime = ((lastTaskToFinish == null) ? dagInfo.getFinishTime() : lastTaskToFinish.getFinishTime()) - ((firstTaskToStart == null) ? dagInfo.getStartTime() : firstTaskToStart.getStartTime());
    return totalTime;
}