Examples with MapReduceApplicationData com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData used on opensource projects

Example 6 with MapReduceApplicationData

use of com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData in project dr-elephant by linkedin.

the class MapperSkewHeuristicTest method analyzeJob.

private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) throws IOException {
    MapReduceCounterData jobCounter = new MapReduceCounterData();
    MapReduceTaskData[] mappers = new MapReduceTaskData[numSmallTasks + numLargeTasks + 1];
    MapReduceCounterData smallCounter = new MapReduceCounterData();
    smallCounter.set(MapReduceCounterData.CounterName.HDFS_BYTES_READ, smallInputSize);
    MapReduceCounterData largeCounter = new MapReduceCounterData();
    largeCounter.set(MapReduceCounterData.CounterName.S3A_BYTES_READ, largeInputSize);
    int i = 0;
    for (; i < numSmallTasks; i++) {
        mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTimeAndCounter(new long[5], smallCounter);
    }
    for (; i < numSmallTasks + numLargeTasks; i++) {
        mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTimeAndCounter(new long[5], largeCounter);
    }
    // Non-sampled task, which does not contain time and counter data
    mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 7 with MapReduceApplicationData

use of com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData in project dr-elephant by linkedin.

the class MapperSkewHeuristicTest method analyzeJobTime.

private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) throws IOException {
    MapReduceTaskData[] mappers = new MapReduceTaskData[numSmallTasks + numLongTasks + 1];
    int i = 0;
    for (; i < numSmallTasks; i++) {
        mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTotalTimeMs(smallTimeTaken, true);
    }
    for (; i < numSmallTasks + numLongTasks; i++) {
        mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTotalTimeMs(longTimeTaken, true);
    }
    // Non-sampled task, which does not contain time data
    mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setMapperData(mappers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 8 with MapReduceApplicationData

use of com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData in project dr-elephant by linkedin.

the class ReducerGCHeuristicTest method analyzeJob.

private Severity analyzeJob(long runtimeMs, long cpuMs, long gcMs) throws IOException {
    MapReduceCounterData jobCounter = new MapReduceCounterData();
    MapReduceTaskData[] reducers = new MapReduceTaskData[NUMTASKS + 1];
    MapReduceCounterData counter = new MapReduceCounterData();
    counter.set(MapReduceCounterData.CounterName.CPU_MILLISECONDS, cpuMs);
    counter.set(MapReduceCounterData.CounterName.GC_MILLISECONDS, gcMs);
    int i = 0;
    for (; i < NUMTASKS; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[] { runtimeMs, 0, 0, 0, 0 }, counter);
    }
    // Non-sampled task, which does not contain time and counter data
    reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setReducerData(reducers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 9 with MapReduceApplicationData

use of com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData in project dr-elephant by linkedin.

the class ReducerSkewHeuristicTest method analyzeJobTime.

private Severity analyzeJobTime(int numSmallTasks, int numLongTasks, long smallTimeTaken, long longTimeTaken) throws IOException {
    MapReduceTaskData[] reducers = new MapReduceTaskData[numSmallTasks + numLongTasks + 1];
    int i = 0;
    for (; i < numSmallTasks; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTotalTimeMs(smallTimeTaken, true);
    }
    for (; i < numSmallTasks + numLongTasks; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTotalTimeMs(longTimeTaken, true);
    }
    // Non-sampled task, which does not contain time data
    reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setReducerData(reducers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 10 with MapReduceApplicationData

use of com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData in project dr-elephant by linkedin.

the class InfoExtractorTest method testLoadInfoMapReduce.

@Test
public void testLoadInfoMapReduce() {
    final String JOB_DEF_URL = "https://grid.example.com:9000/manager?project=project-name&flow=flow-name&job=job-name";
    final String JOB_EXEC_URL = "https://grid.example.com:9000/executor?execid=123456&job=job-name&attempt=0";
    final String FLOW_DEF_URL = "https://grid.example.com:9000/manager?project=project-name&flow=flow-name";
    final String FLOW_EXEC_URL = "https://grid.example.com:9000/executor?execid=123456";
    final String JOB_NAME = "job-name";
    Properties properties = new Properties();
    properties.put(AzkabanScheduler.AZKABAN_JOB_URL, JOB_DEF_URL);
    properties.put(AzkabanScheduler.AZKABAN_ATTEMPT_URL, JOB_EXEC_URL);
    properties.put(AzkabanScheduler.AZKABAN_WORKFLOW_URL, FLOW_DEF_URL);
    properties.put(AzkabanScheduler.AZKABAN_EXECUTION_URL, FLOW_EXEC_URL);
    properties.put(AzkabanScheduler.AZKABAN_JOB_NAME, JOB_NAME);
    AppResult result = new AppResult();
    HadoopApplicationData data = (new MapReduceApplicationData()).setAppId("application_5678").setJobConf(properties);
    InfoExtractor.loadInfo(result, data);
    assertTrue(result.jobDefId.equals(JOB_DEF_URL));
    assertTrue(result.jobExecId.equals(JOB_EXEC_URL));
    assertTrue(result.flowDefId.equals(FLOW_DEF_URL));
    assertTrue(result.flowExecId.equals(FLOW_EXEC_URL));
}