Examples with TezCounterData com.linkedin.drelephant.tez.data.TezCounterData used on opensource projects

Example 1 with TezCounterData

use of com.linkedin.drelephant.tez.data.TezCounterData in project dr-elephant by linkedin.

the class TezTaskLevelAggregatedMetricsTest method testTaskLevelData.

@Test
public void testTaskLevelData() {
    TezTaskData[] taskData = new TezTaskData[3];
    TezCounterData counterData = new TezCounterData();
    counterData.set(TezCounterData.CounterName.PHYSICAL_MEMORY_BYTES, 655577088L);
    counterData.set(TezCounterData.CounterName.VIRTUAL_MEMORY_BYTES, 3051589632L);
    long[] time = { 0, 0, 0, 1464218501117L, 1464218534148L };
    taskData[0] = new TezTaskData("task", "id");
    taskData[0].setTimeAndCounter(time, counterData);
    taskData[1] = new TezTaskData("task", "id");
    taskData[1].setTimeAndCounter(new long[5], counterData);
    // Non-sampled task, which does not contain time and counter data
    taskData[2] = new TezTaskData("task", "id");
    TezTaskLevelAggregatedMetrics taskMetrics = new TezTaskLevelAggregatedMetrics(taskData, 4096L, 1463218501117L);
    Assert.assertEquals(taskMetrics.getDelay(), 1000000000L);
    Assert.assertEquals(taskMetrics.getResourceUsed(), 135168L);
    Assert.assertEquals(taskMetrics.getResourceWasted(), 66627L);
}

Example 2 with TezCounterData

use of com.linkedin.drelephant.tez.data.TezCounterData in project dr-elephant by linkedin.

the class MapperDataSkewHeuristicTest method analyzeJob.

private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) throws IOException {
    TezCounterData jobCounter = new TezCounterData();
    TezTaskData[] mappers = new TezTaskData[numSmallTasks + numLargeTasks + 1];
    TezCounterData smallCounter = new TezCounterData();
    smallCounter.set(TezCounterData.CounterName.HDFS_BYTES_READ, smallInputSize);
    TezCounterData largeCounter = new TezCounterData();
    largeCounter.set(TezCounterData.CounterName.S3A_BYTES_READ, largeInputSize);
    int i = 0;
    for (; i < numSmallTasks; i++) {
        mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTimeAndCounter(new long[5], smallCounter);
    }
    for (; i < numSmallTasks + numLargeTasks; i++) {
        mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTimeAndCounter(new long[5], largeCounter);
    }
    // Non-sampled task, which does not contain time and counter data
    mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
    TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 3 with TezCounterData

use of com.linkedin.drelephant.tez.data.TezCounterData in project dr-elephant by linkedin.

the class MapperGCHeuristicTest method analyzeJob.

private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException {
    TezCounterData jobCounter = new TezCounterData();
    TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1];
    TezCounterData counter = new TezCounterData();
    counter.set(TezCounterData.CounterName.CPU_MILLISECONDS, cpuMs);
    counter.set(TezCounterData.CounterName.GC_TIME_MILLIS, gcMs);
    int i = 0;
    for (; i < NUMTASKS; i++) {
        mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, counter);
    }
    // Non-sampled task, which does not contain time and counter data
    mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
    TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 4 with TezCounterData

use of com.linkedin.drelephant.tez.data.TezCounterData in project dr-elephant by linkedin.

the class MapperSpeedHeuristicTest method analyzeJob.

private Severity analyzeJob(long runtimeMs, long readBytes) throws IOException {
    TezCounterData jobCounter = new TezCounterData();
    TezTaskData[] mappers = new TezTaskData[NUMTASKS + 1];
    TezCounterData counter = new TezCounterData();
    counter.set(TezCounterData.CounterName.HDFS_BYTES_READ, readBytes / 2);
    counter.set(TezCounterData.CounterName.S3A_BYTES_READ, readBytes / 2);
    int i = 0;
    for (; i < NUMTASKS; i++) {
        mappers[i] = new TezTaskData(counter, new long[] { runtimeMs, 0, 0, 0, 0 });
    }
    // Non-sampled task, which does not contain time and counter data
    mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
    TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 5 with TezCounterData

use of com.linkedin.drelephant.tez.data.TezCounterData in project dr-elephant by linkedin.

the class MapperSpillHeuristicTest method analyzeJob.

private Severity analyzeJob(long spilledRecords, long mapRecords, int numTasks) throws IOException {
    TezCounterData jobCounter = new TezCounterData();
    TezTaskData[] mappers = new TezTaskData[numTasks + 1];
    TezCounterData counter = new TezCounterData();
    counter.set(TezCounterData.CounterName.SPILLED_RECORDS, spilledRecords);
    counter.set(TezCounterData.CounterName.OUTPUT_RECORDS, mapRecords);
    int i = 0;
    for (; i < numTasks; i++) {
        mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTimeAndCounter(new long[5], counter);
    }
    // Non-sampled task, which does not contain time and counter data
    mappers[i] = new TezTaskData("task-id-" + i, "task-attempt-id-" + i);
    TezApplicationData data = new TezApplicationData().setCounters(jobCounter).setMapTaskData(mappers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}