Examples with HeuristicResult com.linkedin.drelephant.analysis.HeuristicResult used on opensource projects

Example 26 with HeuristicResult

use of com.linkedin.drelephant.analysis.HeuristicResult in project dr-elephant by linkedin.

the class ReducerTimeHeuristic method apply.

public HeuristicResult apply(TezApplicationData data) {
    if (!data.getSucceeded()) {
        return null;
    }
    TezTaskData[] tasks = data.getReduceTaskData();
    List<Long> runtimesMs = new ArrayList<Long>();
    long taskMinMs = Long.MAX_VALUE;
    long taskMaxMs = 0;
    for (TezTaskData task : tasks) {
        if (task.isSampled()) {
            long taskTime = task.getTotalRunTimeMs();
            runtimesMs.add(taskTime);
            taskMinMs = Math.min(taskMinMs, taskTime);
            taskMaxMs = Math.max(taskMaxMs, taskTime);
        }
    }
    if (taskMinMs == Long.MAX_VALUE) {
        taskMinMs = 0;
    }
    long averageTimeMs = Statistics.average(runtimesMs);
    Severity shortTaskSeverity = shortTaskSeverity(tasks.length, averageTimeMs);
    Severity longTaskSeverity = longTaskSeverity(tasks.length, averageTimeMs);
    Severity severity = Severity.max(shortTaskSeverity, longTaskSeverity);
    HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length));
    result.addResultDetail("Number of tasks", Integer.toString(tasks.length));
    result.addResultDetail("Average task runtime", tasks.length == 0 ? "0" : (Statistics.readableTimespan(averageTimeMs).equals("") ? "0 sec" : Statistics.readableTimespan(averageTimeMs)));
    result.addResultDetail("Max task runtime", tasks.length == 0 ? "0" : (Statistics.readableTimespan(taskMaxMs).equals("") ? "0 sec" : Statistics.readableTimespan(taskMaxMs)));
    result.addResultDetail("Min task runtime", tasks.length == 0 ? "0" : (Statistics.readableTimespan(taskMinMs).equals("") ? "0 sec" : Statistics.readableTimespan(taskMinMs)));
    return result;
}

Example 27 with HeuristicResult

use of com.linkedin.drelephant.analysis.HeuristicResult in project dr-elephant by linkedin.

the class ShuffleSortHeuristicTest method analyzeJob.

private Severity analyzeJob(long shuffleTimeMs, long sortTimeMs, long reduceTimeMs) throws IOException {
    MapReduceCounterData dummyCounter = new MapReduceCounterData();
    MapReduceTaskData[] reducers = new MapReduceTaskData[NUMTASKS + 1];
    int i = 0;
    for (; i < NUMTASKS; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[] { shuffleTimeMs + sortTimeMs + reduceTimeMs, shuffleTimeMs, sortTimeMs, 0, 0 }, dummyCounter);
    }
    // Non-sampled task, which does not contain time and counter data
    reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(dummyCounter).setReducerData(reducers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 28 with HeuristicResult

use of com.linkedin.drelephant.analysis.HeuristicResult in project dr-elephant by linkedin.

the class DistributedCacheLimitHeuristicTest method testHeuristicResult.

/**
 * All cache file sizes are within the limit.
 */
@Test
public void testHeuristicResult() {
    jobConf.setProperty("mapreduce.job.cache.files.filesizes", "100,200,300");
    jobConf.setProperty("mapreduce.job.cache.archives.filesizes", "400,500,600");
    MapReduceApplicationData data = new MapReduceApplicationData().setJobConf(jobConf);
    HeuristicResult result = _heuristic.apply(data);
    assertTrue("Failed to match on expected severity", result.getSeverity() == Severity.NONE);
}

Example 29 with HeuristicResult

use of com.linkedin.drelephant.analysis.HeuristicResult in project dr-elephant by linkedin.

the class DistributedCacheLimitHeuristicTest method testHeuristicResultNoDistributedCacheFiles.

/**
 * Either of the caches are not used by the application.
 */
@Test
public void testHeuristicResultNoDistributedCacheFiles() {
    jobConf.remove("mapreduce.job.cache.files");
    jobConf.remove("mapreduce.job.cache.archives");
    MapReduceApplicationData data = new MapReduceApplicationData().setJobConf(jobConf);
    HeuristicResult result = _heuristic.apply(data);
    assertTrue("Failed to match on expected severity", result == null);
}

Example 30 with HeuristicResult

use of com.linkedin.drelephant.analysis.HeuristicResult in project dr-elephant by linkedin.

the class DistributedCacheLimitHeuristicTest method testHeuristicResultWithEmptyCacheFiles.

/**
 * Cache files are not used by the application.
 */
@Test
public void testHeuristicResultWithEmptyCacheFiles() {
    jobConf.remove("mapreduce.job.cache.files");
    jobConf.setProperty("mapreduce.job.cache.archives.filesizes", "400,500,600");
    MapReduceApplicationData data = new MapReduceApplicationData().setJobConf(jobConf);
    HeuristicResult result = _heuristic.apply(data);
    assertTrue("Failed to match on expected severity", result.getSeverity() == Severity.NONE);
}