Examples with JobIndexInfo org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo used on opensource projects

Example 11 with JobIndexInfo

use of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo in project hadoop by apache.

the class TestFileNameIndexUtils method testJobNamePercentEncoding.

@Test
public void testJobNamePercentEncoding() throws IOException {
    JobIndexInfo info = new JobIndexInfo();
    JobID oldJobId = JobID.forName(JOB_ID);
    JobId jobId = TypeConverter.toYarn(oldJobId);
    info.setJobId(jobId);
    info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
    info.setUser(USER_NAME);
    info.setJobName(JOB_NAME_WITH_DELIMITER);
    info.setFinishTime(Long.parseLong(FINISH_TIME));
    info.setNumMaps(Integer.parseInt(NUM_MAPS));
    info.setNumReduces(Integer.parseInt(NUM_REDUCES));
    info.setJobStatus(JOB_STATUS);
    info.setQueueName(QUEUE_NAME);
    info.setJobStartTime(Long.parseLong(JOB_START_TIME));
    String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
    Assert.assertTrue("Job name not encoded correctly into job history file", jobHistoryFile.contains(JOB_NAME_WITH_DELIMITER_ESCAPE));
}

Example 12 with JobIndexInfo

use of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo in project hadoop by apache.

the class TestFileNameIndexUtils method testUserNamePercentEncoding.

@Test
public void testUserNamePercentEncoding() throws IOException {
    JobIndexInfo info = new JobIndexInfo();
    JobID oldJobId = JobID.forName(JOB_ID);
    JobId jobId = TypeConverter.toYarn(oldJobId);
    info.setJobId(jobId);
    info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
    info.setUser(USER_NAME_WITH_DELIMITER);
    info.setJobName(JOB_NAME);
    info.setFinishTime(Long.parseLong(FINISH_TIME));
    info.setNumMaps(Integer.parseInt(NUM_MAPS));
    info.setNumReduces(Integer.parseInt(NUM_REDUCES));
    info.setJobStatus(JOB_STATUS);
    info.setQueueName(QUEUE_NAME);
    info.setJobStartTime(Long.parseLong(JOB_START_TIME));
    String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info);
    Assert.assertTrue("User name not encoded correctly into job history file", jobHistoryFile.contains(USER_NAME_WITH_DELIMITER_ESCAPE));
}

Example 13 with JobIndexInfo

use of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo in project hadoop by apache.

the class HistoryFileManager method clean.

/**
   * Clean up older history files.
   * 
   * @throws IOException
   *           on any error trying to remove the entries.
   */
@SuppressWarnings("unchecked")
void clean() throws IOException {
    long cutoff = System.currentTimeMillis() - maxHistoryAge;
    boolean halted = false;
    List<FileStatus> serialDirList = getHistoryDirsForCleaning(cutoff);
    // Sort in ascending order. Relies on YYYY/MM/DD/Serial
    Collections.sort(serialDirList);
    for (FileStatus serialDir : serialDirList) {
        List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(serialDir.getPath(), doneDirFc);
        for (FileStatus historyFile : historyFileList) {
            JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(historyFile.getPath().getName());
            long effectiveTimestamp = getEffectiveTimestamp(jobIndexInfo.getFinishTime(), historyFile);
            if (effectiveTimestamp <= cutoff) {
                HistoryFileInfo fileInfo = this.jobListCache.get(jobIndexInfo.getJobId());
                if (fileInfo == null) {
                    String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
                    fileInfo = createHistoryFileInfo(historyFile.getPath(), new Path(historyFile.getPath().getParent(), confFileName), null, jobIndexInfo, true);
                }
                deleteJobFromDone(fileInfo);
            } else {
                halted = true;
                break;
            }
        }
        if (!halted) {
            deleteDir(serialDir);
            removeDirectoryFromSerialNumberIndex(serialDir.getPath());
            existingDoneSubdirs.remove(serialDir.getPath());
        } else {
            // Don't scan any more directories.
            break;
        }
    }
}

Example 14 with JobIndexInfo

use of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo in project hadoop by apache.

the class HistoryFileManager method addDirectoryToJobListCache.

private void addDirectoryToJobListCache(Path path) throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Adding " + path + " to job list cache.");
    }
    List<FileStatus> historyFileList = scanDirectoryForHistoryFiles(path, doneDirFc);
    for (FileStatus fs : historyFileList) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("Adding in history for " + fs.getPath());
        }
        JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
        String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
        String summaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
        HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path(fs.getPath().getParent(), confFileName), new Path(fs.getPath().getParent(), summaryFileName), jobIndexInfo, true);
        jobListCache.addIfAbsent(fileInfo);
    }
}

Example 15 with JobIndexInfo

use of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo in project hadoop by apache.

the class HistoryFileManager method scanIntermediateDirectory.

/**
   * Scans the specified path and populates the intermediate cache.
   * 
   * @param absPath
   * @throws IOException
   */
private void scanIntermediateDirectory(final Path absPath) throws IOException {
    if (LOG.isDebugEnabled()) {
        LOG.debug("Scanning intermediate dir " + absPath);
    }
    List<FileStatus> fileStatusList = scanDirectoryForHistoryFiles(absPath, intermediateDoneDirFc);
    if (LOG.isDebugEnabled()) {
        LOG.debug("Found " + fileStatusList.size() + " files");
    }
    for (FileStatus fs : fileStatusList) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("scanning file: " + fs.getPath());
        }
        JobIndexInfo jobIndexInfo = FileNameIndexUtils.getIndexInfo(fs.getPath().getName());
        String confFileName = JobHistoryUtils.getIntermediateConfFileName(jobIndexInfo.getJobId());
        String summaryFileName = JobHistoryUtils.getIntermediateSummaryFileName(jobIndexInfo.getJobId());
        HistoryFileInfo fileInfo = createHistoryFileInfo(fs.getPath(), new Path(fs.getPath().getParent(), confFileName), new Path(fs.getPath().getParent(), summaryFileName), jobIndexInfo, false);
        final HistoryFileInfo old = jobListCache.addIfAbsent(fileInfo);
        if (old == null || old.didMoveFail()) {
            final HistoryFileInfo found = (old == null) ? fileInfo : old;
            long cutoff = System.currentTimeMillis() - maxHistoryAge;
            if (found.getJobIndexInfo().getFinishTime() <= cutoff) {
                try {
                    found.delete();
                } catch (IOException e) {
                    LOG.warn("Error cleaning up a HistoryFile that is out of date.", e);
                }
            } else {
                if (LOG.isDebugEnabled()) {
                    LOG.debug("Scheduling move to done of " + found);
                }
                moveToDoneExecutor.execute(new Runnable() {

                    @Override
                    public void run() {
                        try {
                            found.moveToDone();
                        } catch (IOException e) {
                            LOG.info("Failed to process fileInfo for job: " + found.getJobId(), e);
                        }
                    }
                });
            }
        } else if (!old.isMovePending()) {
            //This is a duplicate so just delete it
            if (LOG.isDebugEnabled()) {
                LOG.debug("Duplicate: deleting");
            }
            fileInfo.delete();
        }
    }
}