Examples with JobHistoryParser org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser used on opensource projects

Example 11 with JobHistoryParser

use of org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser in project hadoop by apache.

the class TestJobHistoryParsing method testMultipleFailedTasks.

@Test
public void testMultipleFailedTasks() throws Exception {
    JobHistoryParser parser = new JobHistoryParser(Mockito.mock(FSDataInputStream.class));
    EventReader reader = Mockito.mock(EventReader.class);
    // Hack!
    final AtomicInteger numEventsRead = new AtomicInteger(0);
    final org.apache.hadoop.mapreduce.TaskType taskType = org.apache.hadoop.mapreduce.TaskType.MAP;
    final TaskID[] tids = new TaskID[2];
    final JobID jid = new JobID("1", 1);
    tids[0] = new TaskID(jid, taskType, 0);
    tids[1] = new TaskID(jid, taskType, 1);
    Mockito.when(reader.getNextEvent()).thenAnswer(new Answer<HistoryEvent>() {

        public HistoryEvent answer(InvocationOnMock invocation) throws IOException {
            // send two task start and two task fail events for tasks 0 and 1
            int eventId = numEventsRead.getAndIncrement();
            TaskID tid = tids[eventId & 0x1];
            if (eventId < 2) {
                return new TaskStartedEvent(tid, 0, taskType, "");
            }
            if (eventId < 4) {
                TaskFailedEvent tfe = new TaskFailedEvent(tid, 0, taskType, "failed", "FAILED", null, new Counters());
                tfe.setDatum(tfe.getDatum());
                return tfe;
            }
            if (eventId < 5) {
                JobUnsuccessfulCompletionEvent juce = new JobUnsuccessfulCompletionEvent(jid, 100L, 2, 0, "JOB_FAILED", Collections.singletonList("Task failed: " + tids[0].toString()));
                return juce;
            }
            return null;
        }
    });
    JobInfo info = parser.parse(reader);
    assertTrue("Task 0 not implicated", info.getErrorInfo().contains(tids[0].toString()));
}

Example 12 with JobHistoryParser

use of org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser in project hadoop by apache.

the class TestJobHistoryParsing method testFailedJobHistoryWithoutDiagnostics.

@Test
public void testFailedJobHistoryWithoutDiagnostics() throws Exception {
    final Path histPath = new Path(getClass().getClassLoader().getResource("job_1393307629410_0001-1393307687476-user-Sleep+job-1393307723835-0-0-FAILED-default-1393307693920.jhist").getFile());
    final FileSystem lfs = FileSystem.getLocal(new Configuration());
    final FSDataInputStream fsdis = lfs.open(histPath);
    try {
        JobHistoryParser parser = new JobHistoryParser(fsdis);
        JobInfo info = parser.parse();
        assertEquals("History parsed jobId incorrectly", info.getJobId(), JobID.forName("job_1393307629410_0001"));
        assertEquals("Default diagnostics incorrect ", "", info.getErrorInfo());
    } finally {
        fsdis.close();
    }
}

Example 13 with JobHistoryParser

use of org.apache.hadoop.mapreduce.jobhistory.JobHistoryParser in project hadoop by apache.

the class TestJobHistoryParsing method testHistoryParsingForFailedAttempts.

@Test(timeout = 30000)
public void testHistoryParsingForFailedAttempts() throws Exception {
    LOG.info("STARTING testHistoryParsingForFailedAttempts");
    try {
        Configuration conf = new Configuration();
        conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class);
        RackResolver.init(conf);
        MRApp app = new MRAppWithHistoryWithFailedAttempt(2, 1, true, this.getClass().getName(), true);
        app.submit(conf);
        Job job = app.getContext().getAllJobs().values().iterator().next();
        JobId jobId = job.getID();
        app.waitForState(job, JobState.SUCCEEDED);
        // make sure all events are flushed
        app.waitForState(Service.STATE.STOPPED);
        JobHistory jobHistory = new JobHistory();
        jobHistory.init(conf);
        HistoryFileInfo fileInfo = jobHistory.getJobFileInfo(jobId);
        JobHistoryParser parser;
        JobInfo jobInfo;
        synchronized (fileInfo) {
            Path historyFilePath = fileInfo.getHistoryFile();
            FSDataInputStream in = null;
            FileContext fc = null;
            try {
                fc = FileContext.getFileContext(conf);
                in = fc.open(fc.makeQualified(historyFilePath));
            } catch (IOException ioe) {
                LOG.info("Can not open history file: " + historyFilePath, ioe);
                throw (new Exception("Can not open History File"));
            }
            parser = new JobHistoryParser(in);
            jobInfo = parser.parse();
        }
        Exception parseException = parser.getParseException();
        Assert.assertNull("Caught an expected exception " + parseException, parseException);
        int noOffailedAttempts = 0;
        Map<TaskID, TaskInfo> allTasks = jobInfo.getAllTasks();
        for (Task task : job.getTasks().values()) {
            TaskInfo taskInfo = allTasks.get(TypeConverter.fromYarn(task.getID()));
            for (TaskAttempt taskAttempt : task.getAttempts().values()) {
                TaskAttemptInfo taskAttemptInfo = taskInfo.getAllTaskAttempts().get(TypeConverter.fromYarn((taskAttempt.getID())));
                // Verify rack-name for all task attempts
                Assert.assertEquals("rack-name is incorrect", taskAttemptInfo.getRackname(), RACK_NAME);
                if (taskAttemptInfo.getTaskStatus().equals("FAILED")) {
                    noOffailedAttempts++;
                }
            }
        }
        Assert.assertEquals("No of Failed tasks doesn't match.", 2, noOffailedAttempts);
    } finally {
        LOG.info("FINISHED testHistoryParsingForFailedAttempts");
    }
}