Examples with HeuristicResult com.linkedin.drelephant.analysis.HeuristicResult used on opensource projects

Example 16 with HeuristicResult

use of com.linkedin.drelephant.analysis.HeuristicResult in project dr-elephant by linkedin.

the class MapperSkewHeuristicTest method analyzeJobTime.

private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) throws IOException {
    MapReduceTaskData[] mappers = new MapReduceTaskData[numSmallTasks + numLongTasks + 1];
    int i = 0;
    for (; i < numSmallTasks; i++) {
        mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTotalTimeMs(smallTimeTaken, true);
    }
    for (; i < numSmallTasks + numLongTasks; i++) {
        mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTotalTimeMs(longTimeTaken, true);
    }
    // Non-sampled task, which does not contain time data
    mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setMapperData(mappers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 17 with HeuristicResult

use of com.linkedin.drelephant.analysis.HeuristicResult in project dr-elephant by linkedin.

the class ReducerGCHeuristicTest method analyzeJob.

private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException {
    MapReduceCounterData jobCounter = new MapReduceCounterData();
    MapReduceTaskData[] reducers = new MapReduceTaskData[NUMTASKS + 1];
    MapReduceCounterData counter = new MapReduceCounterData();
    counter.set(MapReduceCounterData.CounterName.CPU_MILLISECONDS, cpuMs);
    counter.set(MapReduceCounterData.CounterName.GC_MILLISECONDS, gcMs);
    int i = 0;
    for (; i < NUMTASKS; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, counter);
    }
    // Non-sampled task, which does not contain time and counter data
    reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setReducerData(reducers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 18 with HeuristicResult

use of com.linkedin.drelephant.analysis.HeuristicResult in project dr-elephant by linkedin.

the class ReducerSkewHeuristicTest method analyzeJobTime.

private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) throws IOException {
    MapReduceTaskData[] reducers = new MapReduceTaskData[numSmallTasks + numLongTasks + 1];
    int i = 0;
    for (; i < numSmallTasks; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTotalTimeMs(smallTimeTaken, true);
    }
    for (; i < numSmallTasks + numLongTasks; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTotalTimeMs(longTimeTaken, true);
    }
    // Non-sampled task, which does not contain time data
    reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setReducerData(reducers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 19 with HeuristicResult

use of com.linkedin.drelephant.analysis.HeuristicResult in project dr-elephant by linkedin.

the class DistributedCacheLimitHeuristic method apply.

@Override
public HeuristicResult apply(MapReduceApplicationData data) {
    if (data == null || !data.getSucceeded()) {
        return null;
    }
    Properties jobConf = data.getConf();
    String cacheFiles = jobConf.getProperty(MAPREDUCE_JOB_CACHE_FILES, null);
    String cacheFileSizes = jobConf.getProperty(MAPREDUCE_JOB_CACHE_FILES_FILESIZES, null);
    HeuristicResult result = null;
    if (cacheFiles != null && cacheFileSizes != null) {
        result = new HeuristicResult(_heuristicConfData.getClassName(), _heuristicConfData.getHeuristicName(), Severity.NONE, 0);
        List<String> cacheFilesList = new ArrayList<String>(Arrays.asList(cacheFiles.split(",")));
        List<String> cacheFileSizesList = new ArrayList<String>(Arrays.asList(cacheFileSizes.split(",")));
        int cacheFilesCount = cacheFilesList.size();
        int cacheFileSizesCount = cacheFileSizesList.size();
        if (cacheFilesCount != cacheFileSizesCount) {
            result.setSeverity(Severity.MODERATE);
            logger.warn("Mismatch in the number of files and their corresponding sizes for " + MAPREDUCE_JOB_CACHE_FILES);
            result.addResultDetail(MAPREDUCE_JOB_CACHE_FILES, Integer.toString(cacheFilesCount));
            result.addResultDetail(MAPREDUCE_JOB_CACHE_FILES_FILESIZES, Integer.toString(cacheFileSizesCount));
            return result;
        }
        Map<String, String> cacheFileToSizeMap = new HashMap<String, String>();
        for (int i = 0; i < cacheFilesCount; i++) {
            cacheFileToSizeMap.put(cacheFilesList.get(i), cacheFileSizesList.get(i));
        }
        if (checkFileSizeLimit(result, cacheFileToSizeMap)) {
            result.setSeverity(Severity.CRITICAL);
        }
    }
    String archiveCacheFiles = jobConf.getProperty(MAPREDUCE_JOB_CACHE_ARCHIVES, null);
    String archiveCacheFileSizes = jobConf.getProperty(MAPREDUCE_JOB_CACHE_ARCHIVES_FILESIZES, null);
    if (archiveCacheFiles != null && archiveCacheFileSizes != null) {
        if (result == null) {
            result = new HeuristicResult(_heuristicConfData.getClassName(), _heuristicConfData.getHeuristicName(), Severity.NONE, 0);
        }
        List<String> archiveCacheFilesList = new ArrayList<String>(Arrays.asList(archiveCacheFiles.split(",")));
        List<String> archiveCacheFileSizesList = new ArrayList<String>(Arrays.asList(archiveCacheFileSizes.split(",")));
        int archiveCacheFilesCount = archiveCacheFilesList.size();
        int archiveCacheFileSizesCount = archiveCacheFileSizesList.size();
        if (archiveCacheFilesCount != archiveCacheFileSizesCount) {
            result.setSeverity(Severity.MODERATE);
            logger.warn("Mismatch in the number of files and their corresponding sizes for " + MAPREDUCE_JOB_CACHE_ARCHIVES);
            result.addResultDetail(MAPREDUCE_JOB_CACHE_ARCHIVES, Integer.toString(archiveCacheFilesCount));
            result.addResultDetail(MAPREDUCE_JOB_CACHE_ARCHIVES_FILESIZES, Integer.toString(archiveCacheFileSizesCount));
            return result;
        }
        Map<String, String> archiveCacheFileToSizeMap = new HashMap<String, String>();
        for (int i = 0; i < archiveCacheFilesCount; i++) {
            archiveCacheFileToSizeMap.put(archiveCacheFilesList.get(i), archiveCacheFileSizesList.get(i));
        }
        if (checkFileSizeLimit(result, archiveCacheFileToSizeMap)) {
            result.setSeverity(Severity.CRITICAL);
        }
    }
    return result;
}

Example 20 with HeuristicResult

use of com.linkedin.drelephant.analysis.HeuristicResult in project dr-elephant by linkedin.

the class GenericGCHeuristic method apply.

@Override
public HeuristicResult apply(MapReduceApplicationData data) {
    if (!data.getSucceeded()) {
        return null;
    }
    MapReduceTaskData[] tasks = getTasks(data);
    List<Long> gcMs = new ArrayList<Long>();
    List<Long> cpuMs = new ArrayList<Long>();
    List<Long> runtimesMs = new ArrayList<Long>();
    for (MapReduceTaskData task : tasks) {
        if (task.isTimeAndCounterDataPresent()) {
            runtimesMs.add(task.getTotalRunTimeMs());
            gcMs.add(task.getCounters().get(MapReduceCounterData.CounterName.GC_MILLISECONDS));
            cpuMs.add(task.getCounters().get(MapReduceCounterData.CounterName.CPU_MILLISECONDS));
        }
    }
    long avgRuntimeMs = Statistics.average(runtimesMs);
    long avgCpuMs = Statistics.average(cpuMs);
    long avgGcMs = Statistics.average(gcMs);
    double ratio = avgCpuMs != 0 ? avgGcMs * (1.0) / avgCpuMs : 0;
    Severity severity;
    if (tasks.length == 0) {
        severity = Severity.NONE;
    } else {
        severity = getGcRatioSeverity(avgRuntimeMs, avgCpuMs, avgGcMs);
    }
    HeuristicResult result = new HeuristicResult(_heuristicConfData.getClassName(), _heuristicConfData.getHeuristicName(), severity, Utils.getHeuristicScore(severity, tasks.length));
    result.addResultDetail("Number of tasks", Integer.toString(tasks.length));
    result.addResultDetail("Avg task runtime (ms)", Long.toString(avgRuntimeMs));
    result.addResultDetail("Avg task CPU time (ms)", Long.toString(avgCpuMs));
    result.addResultDetail("Avg task GC time (ms)", Long.toString(avgGcMs));
    result.addResultDetail("Task GC/CPU ratio", Double.toString(ratio));
    return result;
}