Examples with MapReduceApplicationData com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData used on opensource projects

Example 21 with MapReduceApplicationData

use of com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData in project dr-elephant by linkedin.

the class MapperSpillHeuristicTest method analyzeJob.

private Severity analyzeJob(long spilledRecords, long mapRecords, int numTasks) throws IOException {
    MapReduceCounterData jobCounter = new MapReduceCounterData();
    MapReduceTaskData[] mappers = new MapReduceTaskData[numTasks + 1];
    MapReduceCounterData counter = new MapReduceCounterData();
    counter.set(MapReduceCounterData.CounterName.SPILLED_RECORDS, spilledRecords);
    counter.set(MapReduceCounterData.CounterName.MAP_OUTPUT_RECORDS, mapRecords);
    int i = 0;
    for (; i < numTasks; i++) {
        mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTimeAndCounter(new long[5], counter);
    }
    // Non-sampled task, which does not contain time and counter data
    mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 22 with MapReduceApplicationData

use of com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData in project dr-elephant by linkedin.

the class MapperTimeHeuristicTest method analyzeJob.

private Severity analyzeJob(int numTasks, long runtime) throws IOException {
    MapReduceCounterData jobCounter = new MapReduceCounterData();
    MapReduceTaskData[] mappers = new MapReduceTaskData[numTasks + 1];
    MapReduceCounterData taskCounter = new MapReduceCounterData();
    taskCounter.set(MapReduceCounterData.CounterName.HDFS_BYTES_READ, DUMMY_INPUT_SIZE / 4);
    taskCounter.set(MapReduceCounterData.CounterName.S3_BYTES_READ, DUMMY_INPUT_SIZE / 4);
    taskCounter.set(MapReduceCounterData.CounterName.S3A_BYTES_READ, DUMMY_INPUT_SIZE / 4);
    taskCounter.set(MapReduceCounterData.CounterName.S3N_BYTES_READ, DUMMY_INPUT_SIZE / 4);
    int i = 0;
    for (; i < numTasks; i++) {
        mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        mappers[i].setTimeAndCounter(new long[] { runtime, 0, 0, 0, 0 }, taskCounter);
    }
    // Non-sampled task, which does not contain time and counter data
    mappers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 23 with MapReduceApplicationData

use of com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData in project dr-elephant by linkedin.

the class ReducerMemoryHeuristicTest method analyzeJob.

private Severity analyzeJob(long taskAvgMemMB, long containerMemMB) throws IOException {
    MapReduceCounterData jobCounter = new MapReduceCounterData();
    MapReduceTaskData[] reducers = new MapReduceTaskData[NUMTASKS + 1];
    MapReduceCounterData counter = new MapReduceCounterData();
    counter.set(MapReduceCounterData.CounterName.PHYSICAL_MEMORY_BYTES, taskAvgMemMB * FileUtils.ONE_MB);
    Properties p = new Properties();
    p.setProperty(ReducerMemoryHeuristic.REDUCER_MEMORY_CONF, Long.toString(containerMemMB));
    int i = 0;
    for (; i < NUMTASKS; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[5], counter);
    }
    // Non-sampled task, which does not contain time and counter data
    reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setReducerData(reducers);
    data.setJobConf(p);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 24 with MapReduceApplicationData

use of com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData in project dr-elephant by linkedin.

the class ReducerSkewHeuristicTest method analyzeJob.

private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) throws IOException {
    MapReduceCounterData jobCounter = new MapReduceCounterData();
    MapReduceTaskData[] reducers = new MapReduceTaskData[numSmallTasks + numLargeTasks + 1];
    MapReduceCounterData smallCounter = new MapReduceCounterData();
    smallCounter.set(MapReduceCounterData.CounterName.REDUCE_SHUFFLE_BYTES, smallInputSize);
    MapReduceCounterData largeCounter = new MapReduceCounterData();
    largeCounter.set(MapReduceCounterData.CounterName.REDUCE_SHUFFLE_BYTES, largeInputSize);
    int i = 0;
    for (; i < numSmallTasks; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[5], smallCounter);
    }
    for (; i < numSmallTasks + numLargeTasks; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[5], largeCounter);
    }
    // Non-sampled task, which does not contain time and counter data
    reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setReducerData(reducers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 25 with MapReduceApplicationData

use of com.linkedin.drelephant.mapreduce.data.MapReduceApplicationData in project dr-elephant by linkedin.

the class AnalyticJobTest method testGetAnalysis.

@Test
public void testGetAnalysis() throws Exception {
    try {
        // Setup analytic job
        final AnalyticJob analyticJob = new AnalyticJob().setAppId(TEST_JOB_ID1).setAppType(new ApplicationType(TEST_APP_TYPE)).setFinishTime(1462178403).setStartTime(1462178412).setName(TEST_JOB_NAME).setQueueName(TEST_DEFAULT_QUEUE_NAME).setUser(TEST_USERNAME).setTrackingUrl(TEST_TRACKING_URL);
        // Setup job counter data
        String filePath = FILENAME_JOBCOUNTER;
        MapReduceCounterData jobCounter = new MapReduceCounterData();
        setCounterData(jobCounter, filePath);
        // Setup mapper data
        long[][] mapperTasksTime = { { 2563, 0, 0, 0, 0 }, { 2562, 0, 0, 0, 0 }, { 2567, 0, 0, 0, 0 } };
        MapReduceTaskData[] mappers = new MapReduceTaskData[3];
        for (int i = 1; i <= mappers.length; i++) {
            MapReduceCounterData taskCounter = new MapReduceCounterData();
            setCounterData(taskCounter, FILENAME_MAPPERTASK.replaceFirst("\\$", Integer.toString(i)));
            mappers[i - 1] = new MapReduceTaskData("task-id-" + (i - 1), "task-attempt-id-" + (i - 1));
            mappers[i - 1].setTimeAndCounter(mapperTasksTime[i - 1], taskCounter);
        }
        // Setup reducer data
        long[][] reducerTasksTime = { { 1870, 1665, 14, 0, 0 } };
        MapReduceTaskData[] reducers = new MapReduceTaskData[1];
        for (int i = 1; i <= reducers.length; i++) {
            MapReduceCounterData taskCounter = new MapReduceCounterData();
            setCounterData(taskCounter, FILENAME_REDUCERTASK.replaceFirst("\\$", Integer.toString(i)));
            reducers[i - 1] = new MapReduceTaskData("task-id-" + (i - 1), "task-attempt-id-" + (i - 1));
            reducers[i - 1].setTimeAndCounter(reducerTasksTime[i - 1], taskCounter);
        }
        // Setup job configuration data
        filePath = FILENAME_JOBCONF;
        Properties jobConf = TestUtil.loadProperties(filePath);
        // Setup application data
        final MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers).setReducerData(reducers).setJobConf(jobConf).setSucceeded(true).setDiagnosticInfo("").setUsername(TEST_USERNAME).setUrl("").setJobName(TEST_JOB_NAME).setStartTime(1462178412).setFinishTime(1462178403).setRetry(false).setAppId(TEST_JOB_ID1);
        // Setup heuristics
        final List<Heuristic> heuristics = loadHeuristics();
        // Setup job type
        final JobType jobType = new JobType(TEST_JOB_TYPE, TEST_JOBCONF_NAME, TEST_JOBCONF_PATTERN);
        // Set expectations in JMockit
        new Expectations() {

            {
                fetcher.fetchData(analyticJob);
                result = data;
                elephantContext.getHeuristicsForApplicationType(analyticJob.getAppType());
                result = heuristics;
                elephantContext.matchJobType(data);
                result = jobType;
            }
        };
        // Call the method under test
        AppResult result = analyticJob.getAnalysis();
        // Make assertions on result
        assertTrue("Result is null", result != null);
        assertTrue("Score did not match", result.score == TEST_SCORE);
        assertTrue("Severity did not match", result.severity.toString().equals(TEST_SEVERITY));
        assertTrue("APP ID did not match", result.id.equals(TEST_JOB_ID1));
        assertTrue("Scheduler did not match", result.scheduler.equals(TEST_SCHEDULER));
    } catch (Exception e) {
        e.printStackTrace();
        assertFalse("Test failed with exception", true);
    }
}