Search in sources :

Example 1 with MapReduceCounterData

use of com.linkedin.drelephant.mapreduce.data.MapReduceCounterData in project dr-elephant by linkedin.

the class ReducerTimeHeuristicTest method analyzeJob.

private Severity analyzeJob(long runtimeMs, int numTasks) throws IOException {
    MapReduceCounterData dummyCounter = new MapReduceCounterData();
    MapReduceTaskData[] reducers = new MapReduceTaskData[numTasks + 1];
    int i = 0;
    for (; i < numTasks; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, dummyCounter);
    }
    // Non-sampled task, which does not contain time and counter data
    reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(dummyCounter).setReducerData(reducers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}
Also used : MapReduceApplicationData(com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData) MapReduceCounterData(com.linkedin.drelephant.mapreduce.data.MapReduceCounterData) MapReduceTaskData(com.linkedin.drelephant.mapreduce.data.MapReduceTaskData) HeuristicResult(com.linkedin.drelephant.analysis.HeuristicResult)

Example 2 with MapReduceCounterData

use of com.linkedin.drelephant.mapreduce.data.MapReduceCounterData in project dr-elephant by linkedin.

the class TestTaskLevelAggregatedMetrics method testTaskLevelData.

@Test
public void testTaskLevelData() {
    MapReduceTaskData[] taskData = new MapReduceTaskData[3];
    MapReduceCounterData counterData = new MapReduceCounterData();
    counterData.set(MapReduceCounterData.CounterName.PHYSICAL_MEMORY_BYTES, 655577088L);
    counterData.set(MapReduceCounterData.CounterName.VIRTUAL_MEMORY_BYTES, 3051589632L);
    long[] time = { 0, 0, 0, 1464218501117L, 1464218534148L };
    taskData[0] = new MapReduceTaskData("task", "id");
    taskData[0].setTimeAndCounter(time, counterData);
    taskData[1] = new MapReduceTaskData("task", "id");
    taskData[1].setTimeAndCounter(new long[5], counterData);
    // Non-sampled task, which does not contain time and counter data
    taskData[2] = new MapReduceTaskData("task", "id");
    TaskLevelAggregatedMetrics taskMetrics = new TaskLevelAggregatedMetrics(taskData, 4096L, 1463218501117L);
    Assert.assertEquals(taskMetrics.getDelay(), 1000000000L);
    Assert.assertEquals(taskMetrics.getResourceUsed(), 135168L);
    Assert.assertEquals(taskMetrics.getResourceWasted(), 66627L);
}
Also used : MapReduceCounterData(com.linkedin.drelephant.mapreduce.data.MapReduceCounterData) MapReduceTaskData(com.linkedin.drelephant.mapreduce.data.MapReduceTaskData) Test(org.junit.Test)

Example 3 with MapReduceCounterData

use of com.linkedin.drelephant.mapreduce.data.MapReduceCounterData in project dr-elephant by linkedin.

the class MapperSkewHeuristicTest method analyzeJob.

private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) throws IOException {
    MapReduceCounterData jobCounter = new MapReduceCounterData();
    MapReduceTaskData[] mappers = new MapReduceTaskData[numSmallTasks + numLargeTasks + 1];
    MapReduceCounterData smallCounter = new MapReduceCounterData();
    smallCounter.set(MapReduceCounterData.CounterName.HDFS_BYTES_READ, smallInputSize);
    MapReduceCounterData largeCounter = new MapReduceCounterData();
    largeCounter.set(MapReduceCounterData.CounterName.S3A_BYTES_READ, largeInputSize);
    int i = 0;
    for (; i < numSmallTasks; i++) {
        mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTimeAndCounter(new long[5], smallCounter);
    }
    for (; i < numSmallTasks + numLargeTasks; i++) {
        mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTimeAndCounter(new long[5], largeCounter);
    }
    // Non-sampled task, which does not contain time and counter data
    mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}
Also used : MapReduceApplicationData(com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData) MapReduceCounterData(com.linkedin.drelephant.mapreduce.data.MapReduceCounterData) MapReduceTaskData(com.linkedin.drelephant.mapreduce.data.MapReduceTaskData) HeuristicResult(com.linkedin.drelephant.analysis.HeuristicResult)

Example 4 with MapReduceCounterData

use of com.linkedin.drelephant.mapreduce.data.MapReduceCounterData in project dr-elephant by linkedin.

the class ReducerGCHeuristicTest method analyzeJob.

private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException {
    MapReduceCounterData jobCounter = new MapReduceCounterData();
    MapReduceTaskData[] reducers = new MapReduceTaskData[NUMTASKS + 1];
    MapReduceCounterData counter = new MapReduceCounterData();
    counter.set(MapReduceCounterData.CounterName.CPU_MILLISECONDS, cpuMs);
    counter.set(MapReduceCounterData.CounterName.GC_MILLISECONDS, gcMs);
    int i = 0;
    for (; i < NUMTASKS; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, counter);
    }
    // Non-sampled task, which does not contain time and counter data
    reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setReducerData(reducers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}
Also used : MapReduceApplicationData(com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData) MapReduceCounterData(com.linkedin.drelephant.mapreduce.data.MapReduceCounterData) MapReduceTaskData(com.linkedin.drelephant.mapreduce.data.MapReduceTaskData) HeuristicResult(com.linkedin.drelephant.analysis.HeuristicResult)

Example 5 with MapReduceCounterData

use of com.linkedin.drelephant.mapreduce.data.MapReduceCounterData in project dr-elephant by linkedin.

the class ShuffleSortHeuristicTest method analyzeJob.

private Severity analyzeJob(long shuffleTimeMs, long sortTimeMs, long reduceTimeMs) throws IOException {
    MapReduceCounterData dummyCounter = new MapReduceCounterData();
    MapReduceTaskData[] reducers = new MapReduceTaskData[NUMTASKS + 1];
    int i = 0;
    for (; i < NUMTASKS; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[] { shuffleTimeMs + sortTimeMs + reduceTimeMs, shuffleTimeMs, sortTimeMs, 0, 0 }, dummyCounter);
    }
    // Non-sampled task, which does not contain time and counter data
    reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(dummyCounter).setReducerData(reducers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}
Also used : MapReduceApplicationData(com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData) MapReduceCounterData(com.linkedin.drelephant.mapreduce.data.MapReduceCounterData) MapReduceTaskData(com.linkedin.drelephant.mapreduce.data.MapReduceTaskData) HeuristicResult(com.linkedin.drelephant.analysis.HeuristicResult)

Aggregations

MapReduceCounterData (com.linkedin.drelephant.mapreduce.data.MapReduceCounterData)17 MapReduceTaskData (com.linkedin.drelephant.mapreduce.data.MapReduceTaskData)17 MapReduceApplicationData (com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData)15 HeuristicResult (com.linkedin.drelephant.analysis.HeuristicResult)12 Properties (java.util.Properties)6 IOException (java.io.IOException)3 ArrayList (java.util.ArrayList)3 JobHistoryParser (org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser)2 Test (org.junit.Test)2 MapperSkewHeuristic (com.linkedin.drelephant.mapreduce.heuristics.MapperSkewHeuristic)1 MalformedURLException (java.net.MalformedURLException)1 URL (java.net.URL)1 List (java.util.List)1 Map (java.util.Map)1 Expectations (mockit.Expectations)1 AppResult (models.AppResult)1 Configuration (org.apache.hadoop.conf.Configuration)1 Path (org.apache.hadoop.fs.Path)1 TaskAttemptID (org.apache.hadoop.mapreduce.TaskAttemptID)1 TaskID (org.apache.hadoop.mapreduce.TaskID)1