Examples with MapReduceTaskData com.linkedin.drelephant.mapreduce.data.MapReduceTaskData used on opensource projects

Example 21 with MapReduceTaskData

use of com.linkedin.drelephant.mapreduce.data.MapReduceTaskData in project dr-elephant by linkedin.

the class ReducerSkewHeuristicTest method analyzeJob.

private Severity analyzeJob(int numSmallTasks, int numLargeTasks, long smallInputSize, long largeInputSize) throws IOException {
    MapReduceCounterData jobCounter = new MapReduceCounterData();
    MapReduceTaskData[] reducers = new MapReduceTaskData[numSmallTasks + numLargeTasks + 1];
    MapReduceCounterData smallCounter = new MapReduceCounterData();
    smallCounter.set(MapReduceCounterData.CounterName.REDUCE_SHUFFLE_BYTES, smallInputSize);
    MapReduceCounterData largeCounter = new MapReduceCounterData();
    largeCounter.set(MapReduceCounterData.CounterName.REDUCE_SHUFFLE_BYTES, largeInputSize);
    int i = 0;
    for (; i < numSmallTasks; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[5], smallCounter);
    }
    for (; i < numSmallTasks + numLargeTasks; i++) {
        reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
        reducers[i].setTimeAndCounter(new long[5], largeCounter);
    }
    // Non-sampled task, which does not contain time and counter data
    reducers[i] = new MapReduceTaskData("task-id-" + i, "task-attempt-id-" + i);
    MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setReducerData(reducers);
    HeuristicResult result = _heuristic.apply(data);
    return result.getSeverity();
}

Example 22 with MapReduceTaskData

use of com.linkedin.drelephant.mapreduce.data.MapReduceTaskData in project dr-elephant by linkedin.

the class AnalyticJobTest method testGetAnalysis.

@Test
public void testGetAnalysis() throws Exception {
    try {
        // Setup analytic job
        final AnalyticJob analyticJob = new AnalyticJob().setAppId(TEST_JOB_ID1).setAppType(new ApplicationType(TEST_APP_TYPE)).setFinishTime(1462178403).setStartTime(1462178412).setName(TEST_JOB_NAME).setQueueName(TEST_DEFAULT_QUEUE_NAME).setUser(TEST_USERNAME).setTrackingUrl(TEST_TRACKING_URL);
        // Setup job counter data
        String filePath = FILENAME_JOBCOUNTER;
        MapReduceCounterData jobCounter = new MapReduceCounterData();
        setCounterData(jobCounter, filePath);
        // Setup mapper data
        long[][] mapperTasksTime = { { 2563, 0, 0, 0, 0 }, { 2562, 0, 0, 0, 0 }, { 2567, 0, 0, 0, 0 } };
        MapReduceTaskData[] mappers = new MapReduceTaskData[3];
        for (int i = 1; i <= mappers.length; i++) {
            MapReduceCounterData taskCounter = new MapReduceCounterData();
            setCounterData(taskCounter, FILENAME_MAPPERTASK.replaceFirst("\\$", Integer.toString(i)));
            mappers[i - 1] = new MapReduceTaskData("task-id-" + (i - 1), "task-attempt-id-" + (i - 1));
            mappers[i - 1].setTimeAndCounter(mapperTasksTime[i - 1], taskCounter);
        }
        // Setup reducer data
        long[][] reducerTasksTime = { { 1870, 1665, 14, 0, 0 } };
        MapReduceTaskData[] reducers = new MapReduceTaskData[1];
        for (int i = 1; i <= reducers.length; i++) {
            MapReduceCounterData taskCounter = new MapReduceCounterData();
            setCounterData(taskCounter, FILENAME_REDUCERTASK.replaceFirst("\\$", Integer.toString(i)));
            reducers[i - 1] = new MapReduceTaskData("task-id-" + (i - 1), "task-attempt-id-" + (i - 1));
            reducers[i - 1].setTimeAndCounter(reducerTasksTime[i - 1], taskCounter);
        }
        // Setup job configuration data
        filePath = FILENAME_JOBCONF;
        Properties jobConf = TestUtil.loadProperties(filePath);
        // Setup application data
        final MapReduceApplicationData data = new MapReduceApplicationData().setCounters(jobCounter).setMapperData(mappers).setReducerData(reducers).setJobConf(jobConf).setSucceeded(true).setDiagnosticInfo("").setUsername(TEST_USERNAME).setUrl("").setJobName(TEST_JOB_NAME).setStartTime(1462178412).setFinishTime(1462178403).setRetry(false).setAppId(TEST_JOB_ID1);
        // Setup heuristics
        final List<Heuristic> heuristics = loadHeuristics();
        // Setup job type
        final JobType jobType = new JobType(TEST_JOB_TYPE, TEST_JOBCONF_NAME, TEST_JOBCONF_PATTERN);
        // Set expectations in JMockit
        new Expectations() {

            {
                fetcher.fetchData(analyticJob);
                result = data;
                elephantContext.getHeuristicsForApplicationType(analyticJob.getAppType());
                result = heuristics;
                elephantContext.matchJobType(data);
                result = jobType;
            }
        };
        // Call the method under test
        AppResult result = analyticJob.getAnalysis();
        // Make assertions on result
        assertTrue("Result is null", result != null);
        assertTrue("Score did not match", result.score == TEST_SCORE);
        assertTrue("Severity did not match", result.severity.toString().equals(TEST_SEVERITY));
        assertTrue("APP ID did not match", result.id.equals(TEST_JOB_ID1));
        assertTrue("Scheduler did not match", result.scheduler.equals(TEST_SCHEDULER));
    } catch (Exception e) {
        e.printStackTrace();
        assertFalse("Test failed with exception", true);
    }
}

Example 23 with MapReduceTaskData

use of com.linkedin.drelephant.mapreduce.data.MapReduceTaskData in project dr-elephant by linkedin.

the class MapReduceFSFetcherHadoop2 method fetchData.

@Override
public MapReduceApplicationData fetchData(AnalyticJob job) throws IOException {
    DataFiles files = getHistoryFiles(job);
    String confFile = files.getJobConfPath();
    String histFile = files.getJobHistPath();
    String appId = job.getAppId();
    String jobId = Utils.getJobIdFromApplicationId(appId);
    MapReduceApplicationData jobData = new MapReduceApplicationData();
    jobData.setAppId(appId).setJobId(jobId);
    // Fetch job config
    Configuration jobConf = new Configuration(false);
    jobConf.addResource(_fs.open(new Path(confFile)), confFile);
    Properties jobConfProperties = new Properties();
    for (Map.Entry<String, String> entry : jobConf) {
        jobConfProperties.put(entry.getKey(), entry.getValue());
    }
    jobData.setJobConf(jobConfProperties);
    // Check if job history file is too large and should be throttled
    if (_fs.getFileStatus(new Path(histFile)).getLen() > _maxLogSizeInMB * FileUtils.ONE_MB) {
        String errMsg = "The history log of MapReduce application: " + appId + " is over the limit size of " + _maxLogSizeInMB + " MB, the parsing process gets throttled.";
        logger.warn(errMsg);
        jobData.setDiagnosticInfo(errMsg);
        // set succeeded to false to avoid heuristic analysis
        jobData.setSucceeded(false);
        return jobData;
    }
    // Analyze job history file
    JobHistoryParser parser = new JobHistoryParser(_fs, histFile);
    JobHistoryParser.JobInfo jobInfo = parser.parse();
    IOException parseException = parser.getParseException();
    if (parseException != null) {
        throw new RuntimeException("Could not parse history file " + histFile, parseException);
    }
    jobData.setSubmitTime(jobInfo.getSubmitTime());
    jobData.setStartTime(jobInfo.getLaunchTime());
    jobData.setFinishTime(jobInfo.getFinishTime());
    String state = jobInfo.getJobStatus();
    if (state.equals("SUCCEEDED")) {
        jobData.setSucceeded(true);
    } else if (state.equals("FAILED")) {
        jobData.setSucceeded(false);
        jobData.setDiagnosticInfo(jobInfo.getErrorInfo());
    } else {
        throw new RuntimeException("job neither succeeded or failed. can not process it ");
    }
    // Fetch job counter
    MapReduceCounterData jobCounter = getCounterData(jobInfo.getTotalCounters());
    // Fetch task data
    Map<TaskID, JobHistoryParser.TaskInfo> allTasks = jobInfo.getAllTasks();
    List<JobHistoryParser.TaskInfo> mapperInfoList = new ArrayList<JobHistoryParser.TaskInfo>();
    List<JobHistoryParser.TaskInfo> reducerInfoList = new ArrayList<JobHistoryParser.TaskInfo>();
    for (JobHistoryParser.TaskInfo taskInfo : allTasks.values()) {
        if (taskInfo.getTaskType() == TaskType.MAP) {
            mapperInfoList.add(taskInfo);
        } else {
            reducerInfoList.add(taskInfo);
        }
    }
    if (jobInfo.getTotalMaps() > MAX_SAMPLE_SIZE) {
        logger.debug(jobId + " total mappers: " + mapperInfoList.size());
    }
    if (jobInfo.getTotalReduces() > MAX_SAMPLE_SIZE) {
        logger.debug(jobId + " total reducers: " + reducerInfoList.size());
    }
    MapReduceTaskData[] mapperList = getTaskData(jobId, mapperInfoList);
    MapReduceTaskData[] reducerList = getTaskData(jobId, reducerInfoList);
    jobData.setCounters(jobCounter).setMapperData(mapperList).setReducerData(reducerList);
    return jobData;
}

Example 24 with MapReduceTaskData

use of com.linkedin.drelephant.mapreduce.data.MapReduceTaskData in project dr-elephant by linkedin.

the class MapReduceFSFetcherHadoop2 method getTaskData.

protected MapReduceTaskData[] getTaskData(String jobId, List<JobHistoryParser.TaskInfo> infoList) {
    int sampleSize = sampleAndGetSize(jobId, infoList);
    List<MapReduceTaskData> taskList = new ArrayList<MapReduceTaskData>();
    for (int i = 0; i < sampleSize; i++) {
        JobHistoryParser.TaskInfo tInfo = infoList.get(i);
        String taskId = tInfo.getTaskId().toString();
        TaskAttemptID attemptId = null;
        if (tInfo.getTaskStatus().equals("SUCCEEDED")) {
            attemptId = tInfo.getSuccessfulAttemptId();
        } else {
            attemptId = tInfo.getFailedDueToAttemptId();
        }
        MapReduceTaskData taskData = new MapReduceTaskData(taskId, attemptId == null ? "" : attemptId.toString(), tInfo.getTaskStatus());
        MapReduceCounterData taskCounterData = getCounterData(tInfo.getCounters());
        long[] taskExecTime = null;
        if (attemptId != null) {
            taskExecTime = getTaskExecTime(tInfo.getAllTaskAttempts().get(attemptId));
        }
        taskData.setTimeAndCounter(taskExecTime, taskCounterData);
        taskList.add(taskData);
    }
    return taskList.toArray(new MapReduceTaskData[taskList.size()]);
}

Example 25 with MapReduceTaskData

use of com.linkedin.drelephant.mapreduce.data.MapReduceTaskData in project dr-elephant by linkedin.

the class MapReduceFetcherHadoop2 method fetchData.

@Override
public MapReduceApplicationData fetchData(AnalyticJob analyticJob) throws IOException, AuthenticationException {
    String appId = analyticJob.getAppId();
    MapReduceApplicationData jobData = new MapReduceApplicationData();
    String jobId = Utils.getJobIdFromApplicationId(appId);
    jobData.setAppId(appId).setJobId(jobId);
    // Change job tracking url to job history page
    analyticJob.setTrackingUrl(_jhistoryWebAddr + jobId);
    try {
        // Fetch job config
        Properties jobConf = _jsonFactory.getProperties(_urlFactory.getJobConfigURL(jobId));
        jobData.setJobConf(jobConf);
        URL jobURL = _urlFactory.getJobURL(jobId);
        String state = _jsonFactory.getState(jobURL);
        jobData.setSubmitTime(_jsonFactory.getSubmitTime(jobURL));
        jobData.setStartTime(_jsonFactory.getStartTime(jobURL));
        jobData.setFinishTime(_jsonFactory.getFinishTime(jobURL));
        if (state.equals("SUCCEEDED")) {
            jobData.setSucceeded(true);
            // Fetch job counter
            MapReduceCounterData jobCounter = _jsonFactory.getJobCounter(_urlFactory.getJobCounterURL(jobId));
            // Fetch task data
            URL taskListURL = _urlFactory.getTaskListURL(jobId);
            List<MapReduceTaskData> mapperList = new ArrayList<MapReduceTaskData>();
            List<MapReduceTaskData> reducerList = new ArrayList<MapReduceTaskData>();
            _jsonFactory.getTaskDataAll(taskListURL, jobId, mapperList, reducerList);
            MapReduceTaskData[] mapperData = mapperList.toArray(new MapReduceTaskData[mapperList.size()]);
            MapReduceTaskData[] reducerData = reducerList.toArray(new MapReduceTaskData[reducerList.size()]);
            jobData.setCounters(jobCounter).setMapperData(mapperData).setReducerData(reducerData);
        } else if (state.equals("FAILED")) {
            jobData.setSucceeded(false);
            // Fetch job counter
            MapReduceCounterData jobCounter = _jsonFactory.getJobCounter(_urlFactory.getJobCounterURL(jobId));
            // Fetch task data
            URL taskListURL = _urlFactory.getTaskListURL(jobId);
            List<MapReduceTaskData> mapperList = new ArrayList<MapReduceTaskData>();
            List<MapReduceTaskData> reducerList = new ArrayList<MapReduceTaskData>();
            _jsonFactory.getTaskDataAll(taskListURL, jobId, mapperList, reducerList);
            MapReduceTaskData[] mapperData = mapperList.toArray(new MapReduceTaskData[mapperList.size()]);
            MapReduceTaskData[] reducerData = reducerList.toArray(new MapReduceTaskData[reducerList.size()]);
            jobData.setCounters(jobCounter).setMapperData(mapperData).setReducerData(reducerData);
            String diagnosticInfo;
            try {
                diagnosticInfo = parseException(jobData.getJobId(), _jsonFactory.getDiagnosticInfo(jobURL));
            } catch (Exception e) {
                diagnosticInfo = null;
                logger.warn("Failed getting diagnostic info for failed job " + jobData.getJobId());
            }
            jobData.setDiagnosticInfo(diagnosticInfo);
        } else {
            // Should not reach here
            throw new RuntimeException("Job state not supported. Should be either SUCCEEDED or FAILED");
        }
    } finally {
        ThreadContextMR2.updateAuthToken();
    }
    return jobData;
}