use of com.linkedin.drelephant.tez.data.TezApplicationData in project dr-elephant by linkedin.
the class MapperDataSkewHeuristicTest method analyzeJob.
private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) throws IOException {
TezCounterData jobCounter = new TezCounterData();
TezTaskData[] mappers = new TezTaskData[numSmallTasks + numLargeTasks + 1];
TezCounterData smallCounter = new TezCounterData();
smallCounter.set(TezCounterData.CounterName.HDFS_BYTES_READ, smallInputSize);
TezCounterData largeCounter = new TezCounterData();
largeCounter.set(TezCounterData.CounterName.S3A_BYTES_READ, largeInputSize);
int i = 0;
for (; i < numSmallTasks; i++) {
mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
mappers[i].setTimeAndCounter(new long[5], smallCounter);
}
for (; i < numSmallTasks + numLargeTasks; i++) {
mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
mappers[i].setTimeAndCounter(new long[5], largeCounter);
}
// Non-sampled task, which does not contain time and counter data
mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers);
HeuristicResult result = _heuristic.apply(data);
return result.getSeverity();
}
use of com.linkedin.drelephant.tez.data.TezApplicationData in project dr-elephant by linkedin.
the class MapperGCHeuristicTest method analyzeJob.
private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException {
TezCounterData jobCounter = new TezCounterData();
TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1];
TezCounterData counter = new TezCounterData();
counter.set(TezCounterData.CounterName.CPU_MILLISECONDS, cpuMs);
counter.set(TezCounterData.CounterName.GC_TIME_MILLIS, gcMs);
int i = 0;
for (; i < NUMTASKS; i++) {
mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
mappers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, counter);
}
// Non-sampled task, which does not contain time and counter data
mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers);
HeuristicResult result = _heuristic.apply(data);
return result.getSeverity();
}
use of com.linkedin.drelephant.tez.data.TezApplicationData in project dr-elephant by linkedin.
the class MapperSpeedHeuristicTest method analyzeJob.
private Severity analyzeJob(long runtimeMs, long readBytes) throws IOException {
TezCounterData jobCounter = new TezCounterData();
TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1];
TezCounterData counter = new TezCounterData();
counter.set(TezCounterData.CounterName.HDFS_BYTES_READ, readBytes / 2);
counter.set(TezCounterData.CounterName.S3A_BYTES_READ, readBytes / 2);
int i = 0;
for (; i < NUMTASKS; i++) {
mappers[i] = new TezTaskData(counter, new long[] { runtimeMs, 0, 0, 0, 0 });
}
// Non-sampled task, which does not contain time and counter data
mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers);
HeuristicResult result = _heuristic.apply(data);
return result.getSeverity();
}
use of com.linkedin.drelephant.tez.data.TezApplicationData in project dr-elephant by linkedin.
the class MapperSpillHeuristicTest method analyzeJob.
private Severity analyzeJob(long spilledRecords, long mapRecords, int numTasks) throws IOException {
TezCounterData jobCounter = new TezCounterData();
TezTaskData[] mappers = new TezTaskData[numTasks + 1];
TezCounterData counter = new TezCounterData();
counter.set(TezCounterData.CounterName.SPILLED_RECORDS, spilledRecords);
counter.set(TezCounterData.CounterName.OUTPUT_RECORDS, mapRecords);
int i = 0;
for (; i < numTasks; i++) {
mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
mappers[i].setTimeAndCounter(new long[5], counter);
}
// Non-sampled task, which does not contain time and counter data
mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers);
HeuristicResult result = _heuristic.apply(data);
return result.getSeverity();
}
use of com.linkedin.drelephant.tez.data.TezApplicationData in project dr-elephant by linkedin.
the class ReducerDataSkewHeuristicTest method analyzeJob.
private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) throws IOException {
TezCounterData jobCounter = new TezCounterData();
TezTaskData[] reducers = new TezTaskData[numSmallTasks + numLargeTasks + 1];
TezCounterData smallCounter = new TezCounterData();
smallCounter.set(TezCounterData.CounterName.SHUFFLE_BYTES, smallInputSize);
TezCounterData largeCounter = new TezCounterData();
largeCounter.set(TezCounterData.CounterName.SHUFFLE_BYTES, largeInputSize);
int i = 0;
for (; i < numSmallTasks; i++) {
reducers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
reducers[i].setTimeAndCounter(new long[5], smallCounter);
}
for (; i < numSmallTasks + numLargeTasks; i++) {
reducers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
reducers[i].setTimeAndCounter(new long[5], largeCounter);
}
// Non-sampled task, which does not contain time and counter data
reducers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setReduceTaskData(reducers);
HeuristicResult result = _heuristic.apply(data);
return result.getSeverity();
}
Aggregations