Examples with HistoryFileInfo org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo used on opensource projects

Example 21 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class HistoryFileManager method getJobFileInfo.

/**
   * Searches the job history file FileStatus list for the specified JobId.
   * 
   * @param fileStatusList
   *          fileStatus list of Job History Files.
   * @param jobId
   *          The JobId to find.
   * @return A FileInfo object for the jobId, null if not found.
   * @throws IOException
   */
private HistoryFileInfo getJobFileInfo(List<FileStatus> fileStatusList, JobId jobId) throws IOException {
    for (FileStatus fs : fileStatusList) {
        JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
        if (jobIndexInfo.getJobId().equals(jobId)) {
            String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
            String summaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
            HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path(fs.getPath().getParent(), confFileName), new Path(fs.getPath().getParent(), summaryFileName), jobIndexInfo, true);
            return fileInfo;
        }
    }
    return null;
}

Example 22 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class TestJobListCache method testAddExisting.

@Test(timeout = 5000)
public void testAddExisting() {
    JobListCache cache = new JobListCache(2, 1000);
    JobId jobId = MRBuilderUtils.newJobId(1, 1, 1);
    HistoryFileInfo fileInfo = Mockito.mock(HistoryFileInfo.class);
    Mockito.when(fileInfo.getJobId()).thenReturn(jobId);
    cache.addIfAbsent(fileInfo);
    cache.addIfAbsent(fileInfo);
    assertEquals("Incorrect number of cache entries", 1, cache.values().size());
}

Example 23 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class TestUnnecessaryBlockingOnHistoryFileInfo method testTwoThreadsQueryingDifferentJobOfSameUser.

/**
   * This create a test case in which two threads are trying to load two
   * different jobs of the same user under the intermediate directory.
   * One thread should not be blocked by the other thread that is loading
   * a huge job files (This is simulated by hanging up parsing the job files
   * forever). The test will fail by triggering the timeout if one thread is
   * blocked by the other while the other thread is holding the lock on its
   * associated job files and hanging up parsing the files.
   */
@Test(timeout = 600000)
public void testTwoThreadsQueryingDifferentJobOfSameUser() throws InterruptedException, IOException {
    final Configuration config = new Configuration();
    config.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, INTERMEDIATE_DIR.getPath());
    config.setLong(JHAdminConfig.MR_HISTORY_MAX_AGE_MS, Long.MAX_VALUE);
    final JobId job1 = createJobId(0);
    final JobId job2 = createJobId(1);
    final HistoryFileManagerUnderContention historyFileManager = createHistoryFileManager(config, job1, job2);
    Thread webRequest1 = null;
    Thread webRequest2 = null;
    try {
        /**
       * create a dummy .jhist file for job1, and try to load/parse the job
       * files in one child thread.
       */
        createJhistFile(job1);
        webRequest1 = new Thread(new Runnable() {

            @Override
            public void run() {
                try {
                    HistoryFileManager.HistoryFileInfo historyFileInfo = historyFileManager.getFileInfo(job1);
                    historyFileInfo.loadJob();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });
        webRequest1.start();
        historyFileManager.waitUntilIntermediateDirIsScanned(job1);
        /**
       * At this point, thread webRequest1 has finished scanning the
       * intermediate directory and is hanging up parsing the job files while
       * it's holding the lock on the associated HistoryFileInfo object.
       */
        /**
       * create a dummy .jhist file for job2 and try to load/parse the job files
       * in the other child thread. Because job files are not moved from the
       * intermediate directory to the done directory, thread webRequest2
       * will also see the job history files for job1.
       */
        createJhistFile(job2);
        webRequest2 = new Thread(new Runnable() {

            @Override
            public void run() {
                try {
                    HistoryFileManager.HistoryFileInfo historyFileInfo = historyFileManager.getFileInfo(job2);
                    historyFileInfo.loadJob();
                } catch (IOException e) {
                    e.printStackTrace();
                }
            }
        });
        webRequest2.start();
        historyFileManager.waitUntilIntermediateDirIsScanned(job2);
        /**
       * If execution had gotten to this point, then thread webRequest2 would
       * not have tried to acquire the lock of the HistoryFileInfo object
       * associated job1, which is permanently held by thread webRequest1 that
       * is hanging up parsing the job history files, so it was able to proceed
       * with parsing job history files of job2.
       */
        Assert.assertTrue("Thread 2 is blocked while it is trying to " + "load job2 by Thread 1 which is loading job1.", webRequest2.getState() != Thread.State.BLOCKED);
    } finally {
        if (webRequest1 != null) {
            webRequest1.interrupt();
        }
        if (webRequest2 != null) {
            webRequest2.interrupt();
        }
    }
}

Example 24 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class TestJobHistoryParsing method testDiagnosticsForKilledJob.

@Test(timeout = 60000)
public void testDiagnosticsForKilledJob() throws Exception {
    LOG.info("STARTING testDiagnosticsForKilledJob");
    try {
        final Configuration conf = new Configuration();
        conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class);
        RackResolver.init(conf);
        MRApp app = new MRAppWithHistoryWithJobKilled(2, 1, true, this.getClass().getName(), true);
        app.submit(conf);
        Job job = app.getContext().getAllJobs().values().iterator().next();
        JobId jobId = job.getID();
        app.waitForState(job, JobState.KILLED);
        // make sure all events are flushed
        app.waitForState(Service.STATE.STOPPED);
        JobHistory jobHistory = new JobHistory();
        jobHistory.init(conf);
        HistoryFileInfo fileInfo = jobHistory.getJobFileInfo(jobId);
        JobHistoryParser parser;
        JobInfo jobInfo;
        synchronized (fileInfo) {
            Path historyFilePath = fileInfo.getHistoryFile();
            FSDataInputStream in = null;
            FileContext fc = null;
            try {
                fc = FileContext.getFileContext(conf);
                in = fc.open(fc.makeQualified(historyFilePath));
            } catch (IOException ioe) {
                LOG.info("Can not open history file: " + historyFilePath, ioe);
                throw (new Exception("Can not open History File"));
            }
            parser = new JobHistoryParser(in);
            jobInfo = parser.parse();
        }
        Exception parseException = parser.getParseException();
        assertNull("Caught an expected exception " + parseException, parseException);
        final List<String> originalDiagnostics = job.getDiagnostics();
        final String historyError = jobInfo.getErrorInfo();
        assertTrue("No original diagnostics for a failed job", originalDiagnostics != null && !originalDiagnostics.isEmpty());
        assertNotNull("No history error info for a failed job ", historyError);
        for (String diagString : originalDiagnostics) {
            assertTrue(historyError.contains(diagString));
        }
        assertTrue("No killed message in diagnostics", historyError.contains(JobImpl.JOB_KILLED_DIAG));
    } finally {
        LOG.info("FINISHED testDiagnosticsForKilledJob");
    }
}

Example 25 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class TestJobHistoryParsing method testCountersForFailedTask.

@Test(timeout = 60000)
public void testCountersForFailedTask() throws Exception {
    LOG.info("STARTING testCountersForFailedTask");
    try {
        Configuration conf = new Configuration();
        conf.setClass(NET_TOPOLOGY_NODE_SWITCH_MAPPING_IMPL_KEY, MyResolver.class, DNSToSwitchMapping.class);
        RackResolver.init(conf);
        MRApp app = new MRAppWithHistoryWithFailedTask(2, 1, true, this.getClass().getName(), true);
        app.submit(conf);
        Job job = app.getContext().getAllJobs().values().iterator().next();
        JobId jobId = job.getID();
        app.waitForState(job, JobState.FAILED);
        // make sure all events are flushed
        app.waitForState(Service.STATE.STOPPED);
        JobHistory jobHistory = new JobHistory();
        jobHistory.init(conf);
        HistoryFileInfo fileInfo = jobHistory.getJobFileInfo(jobId);
        JobHistoryParser parser;
        JobInfo jobInfo;
        synchronized (fileInfo) {
            Path historyFilePath = fileInfo.getHistoryFile();
            FSDataInputStream in = null;
            FileContext fc = null;
            try {
                fc = FileContext.getFileContext(conf);
                in = fc.open(fc.makeQualified(historyFilePath));
            } catch (IOException ioe) {
                LOG.info("Can not open history file: " + historyFilePath, ioe);
                throw (new Exception("Can not open History File"));
            }
            parser = new JobHistoryParser(in);
            jobInfo = parser.parse();
        }
        Exception parseException = parser.getParseException();
        Assert.assertNull("Caught an expected exception " + parseException, parseException);
        for (Map.Entry<TaskID, TaskInfo> entry : jobInfo.getAllTasks().entrySet()) {
            TaskId yarnTaskID = TypeConverter.toYarn(entry.getKey());
            CompletedTask ct = new CompletedTask(yarnTaskID, entry.getValue());
            Assert.assertNotNull("completed task report has null counters", ct.getReport().getCounters());
        }
        final List<String> originalDiagnostics = job.getDiagnostics();
        final String historyError = jobInfo.getErrorInfo();
        assertTrue("No original diagnostics for a failed job", originalDiagnostics != null && !originalDiagnostics.isEmpty());
        assertNotNull("No history error info for a failed job ", historyError);
        for (String diagString : originalDiagnostics) {
            assertTrue(historyError.contains(diagString));
        }
    } finally {
        LOG.info("FINISHED testCountersForFailedTask");
    }
}