Examples with MRJobStats org.apache.pig.tools.pigstats.mapreduce.MRJobStats used on opensource projects

Example 1 with MRJobStats

use of org.apache.pig.tools.pigstats.mapreduce.MRJobStats in project ambrose by twitter.

the class PigJob method setJobStats.

@JsonIgnore
public void setJobStats(JobStats stats) {
    this.inputInfoList = inputInfoList(stats.getInputs());
    this.outputInfoList = outputInfoList(stats.getOutputs());
    // job metrics
    Map<String, Number> metrics = Maps.newHashMap();
    metrics.put("hdfsBytesRead", stats.getHdfsBytesRead());
    metrics.put("hdfsBytesWritten", stats.getHdfsBytesWritten());
    metrics.put("bytesWritten", stats.getBytesWritten());
    metrics.put("recordWritten", stats.getRecordWrittern());
    if (stats instanceof MRJobStats) {
        MRJobStats mrStats = (MRJobStats) stats;
        setCounterGroupMap(CounterGroup.counterGroupsByName(mrStats.getHadoopCounters()));
        metrics.put("avgMapTime", mrStats.getAvgMapTime());
        // internal pig seems to have fixed typo in OSS pig method name; avoid NoSuchMethodException
        // TODO: Remove this once internal pig is replaced w/ OSS pig
        Number avgReduceTime = null;
        try {
            Method method = mrStats.getClass().getMethod("getAvgReduceTime");
            avgReduceTime = (Number) method.invoke(mrStats);
        } catch (NoSuchMethodException e) {
        // assume we're dealing with OSS pig; ignore
        } catch (InvocationTargetException e) {
            throw new RuntimeException("Failed to invoke MRJobStats.getAvgReduceTime", e);
        } catch (IllegalAccessException e) {
            throw new RuntimeException("Failed to invoke MRJobStats.getAvgReduceTime", e);
        }
        if (avgReduceTime == null) {
            avgReduceTime = mrStats.getAvgREduceTime();
        }
        metrics.put("avgReduceTime", avgReduceTime);
        metrics.put("mapInputRecords", mrStats.getMapInputRecords());
        metrics.put("mapOutputRecords", mrStats.getMapOutputRecords());
        metrics.put("maxMapTime", mrStats.getMaxMapTime());
        metrics.put("maxReduceTime", mrStats.getMaxReduceTime());
        metrics.put("minMapTime", mrStats.getMinMapTime());
        metrics.put("minReduceTime", mrStats.getMinReduceTime());
        metrics.put("numberMaps", mrStats.getNumberMaps());
        metrics.put("numberReduces", mrStats.getNumberReduces());
        metrics.put("proactiveSpillCountObjects", mrStats.getProactiveSpillCountObjects());
        metrics.put("proactiveSpillCountRecs", mrStats.getProactiveSpillCountRecs());
        metrics.put("reduceInputRecords", mrStats.getReduceInputRecords());
        metrics.put("reduceOutputRecords", mrStats.getReduceOutputRecords());
        metrics.put("SMMSpillCount", mrStats.getSMMSpillCount());
    }
    setMetrics(metrics);
}