Examples with ATSFileParser org.apache.tez.history.parser.ATSFileParser used on opensource projects

Example 1 with ATSFileParser

use of org.apache.tez.history.parser.ATSFileParser in project tez by apache.

the class TezAnalyzerBase method run.

@Override
public int run(String[] args) throws Exception {
    // Parse downloaded contents
    CommandLine cmdLine = null;
    try {
        cmdLine = new GnuParser().parse(buildOptions(), args);
    } catch (ParseException e) {
        System.err.println("Invalid options on command line");
        printUsage();
        return -1;
    }
    saveResults = cmdLine.hasOption(SAVE_RESULTS);
    if (cmdLine.hasOption(HELP)) {
        printUsage();
        return 0;
    }
    outputDir = cmdLine.getOptionValue(OUTPUT_DIR);
    if (outputDir == null) {
        outputDir = System.getProperty("user.dir");
    }
    File file = null;
    if (cmdLine.hasOption(EVENT_FILE_NAME)) {
        file = new File(cmdLine.getOptionValue(EVENT_FILE_NAME));
    }
    String dagId = cmdLine.getOptionValue(DAG_ID);
    DagInfo dagInfo = null;
    if (file == null) {
        if (cmdLine.hasOption(FROM_SIMPLE_HISTORY)) {
            System.err.println("Event file name must be specified when using simple history");
            printUsage();
            return -2;
        }
        // using ATS - try to download directly
        String[] importArgs = { "--dagId=" + dagId, "--downloadDir=" + outputDir };
        int result = ATSImportTool.process(importArgs);
        if (result != 0) {
            System.err.println("Error downloading data from ATS");
            return -3;
        }
        // Parse ATS data and verify results
        // Parse downloaded contents
        file = new File(outputDir + Path.SEPARATOR + dagId + ".zip");
    }
    Preconditions.checkState(file != null);
    if (!cmdLine.hasOption(FROM_SIMPLE_HISTORY)) {
        ATSFileParser parser = new ATSFileParser(file);
        dagInfo = parser.getDAGData(dagId);
    } else {
        SimpleHistoryParser parser = new SimpleHistoryParser(file);
        dagInfo = parser.getDAGData(dagId);
    }
    Preconditions.checkState(dagInfo.getDagId().equals(dagId));
    analyze(dagInfo);
    Result result = getResult();
    if (saveResults && (result instanceof CSVResult)) {
        String fileName = outputDir + File.separator + this.getClass().getName() + "_" + dagInfo.getDagId() + ".csv";
        ((CSVResult) result).dumpToFile(fileName);
        LOG.info("Saved results in " + fileName);
    }
    return 0;
}

Example 2 with ATSFileParser

use of org.apache.tez.history.parser.ATSFileParser in project tez by apache.

the class TestAnalyzer method getDagInfo.

private DagInfo getDagInfo(String dagId) throws Exception {
    // sleep for a bit to let ATS events be sent from AM
    DagInfo dagInfo = null;
    if (usingATS) {
        // Export the data from ATS
        String[] args = { "--dagId=" + dagId, "--downloadDir=" + DOWNLOAD_DIR, "--yarnTimelineAddress=" + yarnTimelineAddress };
        int result = ATSImportTool.process(args);
        assertTrue(result == 0);
        // Parse ATS data and verify results
        // Parse downloaded contents
        File downloadedFile = new File(DOWNLOAD_DIR + Path.SEPARATOR + dagId + ".zip");
        ATSFileParser parser = new ATSFileParser(downloadedFile);
        dagInfo = parser.getDAGData(dagId);
        assertTrue(dagInfo.getDagId().equals(dagId));
    } else {
        if (!downloadedSimpleHistoryFile) {
            downloadedSimpleHistoryFile = true;
            TezDAGID tezDAGID = TezDAGID.fromString(dagId);
            ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(tezDAGID.getApplicationId(), 1);
            Path historyPath = new Path(miniTezCluster.getConfig().get("fs.defaultFS") + SIMPLE_HISTORY_DIR + HISTORY_TXT + "." + applicationAttemptId);
            FileSystem fs = historyPath.getFileSystem(miniTezCluster.getConfig());
            Path localPath = new Path(DOWNLOAD_DIR, HISTORY_TXT);
            fs.copyToLocalFile(historyPath, localPath);
        }
        // Now parse via SimpleHistory
        File localFile = new File(DOWNLOAD_DIR, HISTORY_TXT);
        SimpleHistoryParser parser = new SimpleHistoryParser(localFile);
        dagInfo = parser.getDAGData(dagId);
        assertTrue(dagInfo.getDagId().equals(dagId));
    }
    return dagInfo;
}

Example 3 with ATSFileParser

use of org.apache.tez.history.parser.ATSFileParser in project tez by apache.

the class TestHistoryParser method getDagInfo.

private DagInfo getDagInfo(String dagId) throws TezException {
    // Parse downloaded contents
    File downloadedFile = new File(DOWNLOAD_DIR + Path.SEPARATOR + dagId + ".zip");
    ATSFileParser parser = new ATSFileParser(downloadedFile);
    DagInfo dagInfo = parser.getDAGData(dagId);
    assertTrue(dagInfo.getDagId().equals(dagId));
    return dagInfo;
}