Examples with HistoryFileInfo org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo used on opensource projects

Example 16 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class HistoryFileManager method addDirectoryToJobListCache.

private void addDirectoryToJobListCache(Path path) throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Adding " + path + " to job list cache.");
    }
    List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path, doneDirFc);
    for (FileStatus fs : historyFileList) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding in history for " + fs.getPath());
        }
        JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
        String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
        String summaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
        HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path(fs.getPath().getParent(), confFileName), new Path(fs.getPath().getParent(), summaryFileName), jobIndexInfo, true);
        jobListCache.addIfAbsent(fileInfo);
    }
}

Example 17 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class HistoryFileManager method scanIntermediateDirectory.

/**
   * Scans the specified path and populates the intermediate cache.
   * 
   * @param absPath
   * @throws IOException
   */
private void scanIntermediateDirectory(final Path absPath) throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Scanning intermediate dir " + absPath);
    }
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(absPath, intermediateDoneDirFc);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Found " + fileStatusList.size() + " files");
    }
    for (FileStatus fs : fileStatusList) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("scanning file: " + fs.getPath());
        }
        JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
        String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
        String summaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
        HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path(fs.getPath().getParent(), confFileName), new Path(fs.getPath().getParent(), summaryFileName), jobIndexInfo, false);
        final HistoryFileInfo old = jobListCache.addIfAbsent(fileInfo);
        if (old == null || old.didMoveFail()) {
            final HistoryFileInfo found = (old == null) ? fileInfo : old;
            long cutoff = System.currentTimeMillis() - maxHistoryAge;
            if (found.getJobIndexInfo().getFinishTime() <= cutoff) {
                try {
                    found.delete();
                } catch (IOException e) {
                    LOG.warn("Error cleaning up a HistoryFile that is out of date.", e);
                }
            } else {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Scheduling move to done of " + found);
                }
                moveToDoneExecutor.execute(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            found.moveToDone();
                        } catch (IOException e) {
                            LOG.info("Failed to process fileInfo for job: " + found.getJobId(), e);
                        }
                    }
                });
            }
        } else if (!old.isMovePending()) {
            //This is a duplicate so just delete it
            if (LOG.isDebugEnabled()) {
                LOG.debug("Duplicate: deleting");
            }
            fileInfo.delete();
        }
    }
}

Example 18 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class TestHistoryFileManager method testHistoryFileInfoLoadNormalSizedJobShouldReturnCompletedJob.

@Test
public void testHistoryFileInfoLoadNormalSizedJobShouldReturnCompletedJob() throws Exception {
    HistoryFileManagerTest hmTest = new HistoryFileManagerTest();
    final int numOfTasks = 100;
    Configuration conf = dfsCluster.getConfiguration(0);
    conf.setInt(JHAdminConfig.MR_HS_LOADED_JOBS_TASKS_MAX, numOfTasks + numOfTasks + 1);
    hmTest.init(conf);
    // set up a job of which the number of tasks is smaller than the maximum
    // allowed, and therefore will be fully loaded.
    final String jobId = "job_1416424547277_0002";
    JobIndexInfo jobIndexInfo = new JobIndexInfo();
    jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(jobId)));
    jobIndexInfo.setNumMaps(numOfTasks);
    jobIndexInfo.setNumReduces(numOfTasks);
    final String historyFile = getClass().getClassLoader().getResource("job_2.0.3-alpha-FAILED.jhist").getFile();
    final Path historyFilePath = FileSystem.getLocal(conf).makeQualified(new Path(historyFile));
    HistoryFileInfo info = hmTest.getHistoryFileInfo(historyFilePath, null, null, jobIndexInfo, false);
    Job job = info.loadJob();
    Assert.assertTrue("Should return an instance of CompletedJob as " + "a result of parsing the job history file of the job", job instanceof CompletedJob);
}

Example 19 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class TestHistoryFileManager method testHistoryFileInfoLoadOversizedJobShouldReturnUnParsedJob.

@Test
public void testHistoryFileInfoLoadOversizedJobShouldReturnUnParsedJob() throws Exception {
    HistoryFileManagerTest hmTest = new HistoryFileManagerTest();
    int allowedMaximumTasks = 5;
    Configuration conf = dfsCluster.getConfiguration(0);
    conf.setInt(JHAdminConfig.MR_HS_LOADED_JOBS_TASKS_MAX, allowedMaximumTasks);
    hmTest.init(conf);
    // set up a job of which the number of tasks is greater than maximum allowed
    String jobId = "job_1410889000000_123456";
    JobIndexInfo jobIndexInfo = new JobIndexInfo();
    jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(jobId)));
    jobIndexInfo.setNumMaps(allowedMaximumTasks);
    jobIndexInfo.setNumReduces(allowedMaximumTasks);
    HistoryFileInfo info = hmTest.getHistoryFileInfo(null, null, null, jobIndexInfo, false);
    Job job = info.loadJob();
    Assert.assertTrue("Should return an instance of UnparsedJob to indicate" + " the job history file is not parsed", job instanceof UnparsedJob);
}

Example 20 with HistoryFileInfo

use of org.apache.hadoop.mapreduce.v2.hs.HistoryFileManager.HistoryFileInfo in project hadoop by apache.

the class CachedHistoryStorage method loadJob.

private Job loadJob(JobId jobId) throws RuntimeException, IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Looking for Job " + jobId);
    }
    HistoryFileInfo fileInfo;
    fileInfo = hsManager.getFileInfo(jobId);
    if (fileInfo == null) {
        throw new HSFileRuntimeException("Unable to find job " + jobId);
    } else if (fileInfo.isDeleted()) {
        throw new HSFileRuntimeException("Cannot load deleted job " + jobId);
    } else {
        return fileInfo.loadJob();
    }
}