use of com.linkedin.drelephant.mapreduce.data.MapReduceCounterData in project dr-elephant by linkedin.
the class ReducerTimeHeuristicTest method analyzeJob.
private Severity analyzeJob(long runtimeMs, int numTasks) throws IOException {
MapReduceCounterData dummyCounter = new MapReduceCounterData();
MapReduceTaskData[] reducers = new MapReduceTaskData[numTasks + 1];
int i = 0;
for (; i < numTasks; i++) {
reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, dummyCounter);
}
// Non-sampled task, which does not contain time and counter data
reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
MapReduceApplicationData data = new MapReduceApplicationData().setCounters(dummyCounter).setReducerData(reducers);
HeuristicResult result = _heuristic.apply(data);
return result.getSeverity();
}
use of com.linkedin.drelephant.mapreduce.data.MapReduceCounterData in project dr-elephant by linkedin.
the class TestTaskLevelAggregatedMetrics method testTaskLevelData.
@Test
public void testTaskLevelData() {
MapReduceTaskData[] taskData = new MapReduceTaskData[3];
MapReduceCounterData counterData = new MapReduceCounterData();
counterData.set(MapReduceCounterData.CounterName.PHYSICAL_MEMORY_BYTES, 655577088L);
counterData.set(MapReduceCounterData.CounterName.VIRTUAL_MEMORY_BYTES, 3051589632L);
long[] time = { 0, 0, 0, 1464218501117L, 1464218534148L };
taskData[0] = new MapReduceTaskData("task", "id");
taskData[0].setTimeAndCounter(time, counterData);
taskData[1] = new MapReduceTaskData("task", "id");
taskData[1].setTimeAndCounter(new long[5], counterData);
// Non-sampled task, which does not contain time and counter data
taskData[2] = new MapReduceTaskData("task", "id");
TaskLevelAggregatedMetrics taskMetrics = new TaskLevelAggregatedMetrics(taskData, 4096L, 1463218501117L);
Assert.assertEquals(taskMetrics.getDelay(), 1000000000L);
Assert.assertEquals(taskMetrics.getResourceUsed(), 135168L);
Assert.assertEquals(taskMetrics.getResourceWasted(), 66627L);
}
use of com.linkedin.drelephant.mapreduce.data.MapReduceCounterData in project dr-elephant by linkedin.
the class MapperSkewHeuristicTest method analyzeJob.
private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) throws IOException {
MapReduceCounterData jobCounter = new MapReduceCounterData();
MapReduceTaskData[] mappers = new MapReduceTaskData[numSmallTasks + numLargeTasks + 1];
MapReduceCounterData smallCounter = new MapReduceCounterData();
smallCounter.set(MapReduceCounterData.CounterName.HDFS_BYTES_READ, smallInputSize);
MapReduceCounterData largeCounter = new MapReduceCounterData();
largeCounter.set(MapReduceCounterData.CounterName.S3A_BYTES_READ, largeInputSize);
int i = 0;
for (; i < numSmallTasks; i++) {
mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
mappers[i].setTimeAndCounter(new long[5], smallCounter);
}
for (; i < numSmallTasks + numLargeTasks; i++) {
mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
mappers[i].setTimeAndCounter(new long[5], largeCounter);
}
// Non-sampled task, which does not contain time and counter data
mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers);
HeuristicResult result = _heuristic.apply(data);
return result.getSeverity();
}
use of com.linkedin.drelephant.mapreduce.data.MapReduceCounterData in project dr-elephant by linkedin.
the class ReducerGCHeuristicTest method analyzeJob.
private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException {
MapReduceCounterData jobCounter = new MapReduceCounterData();
MapReduceTaskData[] reducers = new MapReduceTaskData[NUMTASKS + 1];
MapReduceCounterData counter = new MapReduceCounterData();
counter.set(MapReduceCounterData.CounterName.CPU_MILLISECONDS, cpuMs);
counter.set(MapReduceCounterData.CounterName.GC_MILLISECONDS, gcMs);
int i = 0;
for (; i < NUMTASKS; i++) {
reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, counter);
}
// Non-sampled task, which does not contain time and counter data
reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setReducerData(reducers);
HeuristicResult result = _heuristic.apply(data);
return result.getSeverity();
}
use of com.linkedin.drelephant.mapreduce.data.MapReduceCounterData in project dr-elephant by linkedin.
the class ShuffleSortHeuristicTest method analyzeJob.
private Severity analyzeJob(long shuffleTimeMs, long sortTimeMs, long reduceTimeMs) throws IOException {
MapReduceCounterData dummyCounter = new MapReduceCounterData();
MapReduceTaskData[] reducers = new MapReduceTaskData[NUMTASKS + 1];
int i = 0;
for (; i < NUMTASKS; i++) {
reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
reducers[i].setTimeAndCounter(new long[] { shuffleTimeMs + sortTimeMs + reduceTimeMs, shuffleTimeMs, sortTimeMs, 0, 0 }, dummyCounter);
}
// Non-sampled task, which does not contain time and counter data
reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
MapReduceApplicationData data = new MapReduceApplicationData().setCounters(dummyCounter).setReducerData(reducers);
HeuristicResult result = _heuristic.apply(data);
return result.getSeverity();
}
Aggregations