Examples with FSStatsAggregator org.apache.hadoop.hive.ql.stats.fs.FSStatsAggregator used on opensource projects

Example 1 with FSStatsAggregator

use of org.apache.hadoop.hive.ql.stats.fs.FSStatsAggregator in project hive by apache.

the class ExplainSemanticAnalyzer method aggregateStats.

private Map<String, Long> aggregateStats(Path localTmpPath) {
    Map<String, Long> opIdToRuntimeNumRows = new HashMap<String, Long>();
    // localTmpPath is the root of all the stats.
    // Under it, there will be SEL_1/statsfiles, SEL_2/statsfiles etc where SEL_1 and SEL_2 are the op ids.
    FileSystem fs;
    FileStatus[] statuses = null;
    try {
        fs = localTmpPath.getFileSystem(conf);
        statuses = fs.listStatus(localTmpPath, FileUtils.HIDDEN_FILES_PATH_FILTER);
    // statuses can be null if it is DDL, etc
    } catch (IOException e) {
        LOG.warn(e.toString());
    }
    if (statuses != null) {
        for (FileStatus status : statuses) {
            if (status.isDir()) {
                StatsCollectionContext scc = new StatsCollectionContext(conf);
                String[] names = status.getPath().toString().split(Path.SEPARATOR);
                String opId = names[names.length - 1];
                scc.setStatsTmpDir(status.getPath().toString());
                StatsAggregator statsAggregator = new FSStatsAggregator();
                if (!statsAggregator.connect(scc)) {
                    // -1 means that there is no stats
                    opIdToRuntimeNumRows.put(opId, -1L);
                } else {
                    String value = statsAggregator.aggregateStats("", StatsSetupConst.RUN_TIME_ROW_COUNT);
                    opIdToRuntimeNumRows.put(opId, Long.parseLong(value));
                }
                if (statsAggregator != null) {
                    statsAggregator.closeConnection(scc);
                }
            }
        }
    }
    return opIdToRuntimeNumRows;
}