Examples with JobIndexInfo org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo used on opensource projects

Example 6 with JobIndexInfo

use of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo in project hadoop by apache.

the class TestHistoryFileManager method testHistoryFileInfoSummaryFileNotExist.

@Test
public void testHistoryFileInfoSummaryFileNotExist() throws Exception {
    HistoryFileManagerTest hmTest = new HistoryFileManagerTest();
    String job = "job_1410889000000_123456";
    Path summaryFile = new Path(job + ".summary");
    JobIndexInfo jobIndexInfo = new JobIndexInfo();
    jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(job)));
    Configuration conf = dfsCluster.getConfiguration(0);
    conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, "/" + UUID.randomUUID());
    conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, "/" + UUID.randomUUID());
    hmTest.serviceInit(conf);
    HistoryFileInfo info = hmTest.getHistoryFileInfo(null, null, summaryFile, jobIndexInfo, false);
    info.moveToDone();
    Assert.assertFalse(info.didMoveFail());
}

Example 7 with JobIndexInfo

use of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo in project hadoop by apache.

the class TestHistoryFileManager method testHistoryFileInfoShouldReturnCompletedJobIfMaxNotConfiged.

@Test
public void testHistoryFileInfoShouldReturnCompletedJobIfMaxNotConfiged() throws Exception {
    HistoryFileManagerTest hmTest = new HistoryFileManagerTest();
    Configuration conf = dfsCluster.getConfiguration(0);
    conf.setInt(JHAdminConfig.MR_HS_LOADED_JOBS_TASKS_MAX, -1);
    hmTest.init(conf);
    final String jobId = "job_1416424547277_0002";
    JobIndexInfo jobIndexInfo = new JobIndexInfo();
    jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(jobId)));
    jobIndexInfo.setNumMaps(100);
    jobIndexInfo.setNumReduces(100);
    final String historyFile = getClass().getClassLoader().getResource("job_2.0.3-alpha-FAILED.jhist").getFile();
    final Path historyFilePath = FileSystem.getLocal(conf).makeQualified(new Path(historyFile));
    HistoryFileInfo info = hmTest.getHistoryFileInfo(historyFilePath, null, null, jobIndexInfo, false);
    Job job = info.loadJob();
    Assert.assertTrue("Should return an instance of CompletedJob as " + "a result of parsing the job history file of the job", job instanceof CompletedJob);
}

Example 8 with JobIndexInfo

use of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo in project hadoop by apache.

the class FileNameIndexUtils method getIndexInfo.

/**
   * Parses the provided job history file name to construct a
   * JobIndexInfo object which is returned.
   * 
   * @param jhFileName the job history filename.
   * @return a JobIndexInfo object built from the filename.
   */
public static JobIndexInfo getIndexInfo(String jhFileName) throws IOException {
    String fileName = jhFileName.substring(0, jhFileName.indexOf(JobHistoryUtils.JOB_HISTORY_FILE_EXTENSION));
    JobIndexInfo indexInfo = new JobIndexInfo();
    String[] jobDetails = fileName.split(DELIMITER);
    JobID oldJobId = JobID.forName(decodeJobHistoryFileName(jobDetails[JOB_ID_INDEX]));
    JobId jobId = TypeConverter.toYarn(oldJobId);
    indexInfo.setJobId(jobId);
    // Do not fail if there are some minor parse errors
    try {
        try {
            indexInfo.setSubmitTime(Long.parseLong(decodeJobHistoryFileName(jobDetails[SUBMIT_TIME_INDEX])));
        } catch (NumberFormatException e) {
            LOG.warn("Unable to parse submit time from job history file " + jhFileName + " : " + e);
        }
        indexInfo.setUser(decodeJobHistoryFileName(jobDetails[USER_INDEX]));
        indexInfo.setJobName(decodeJobHistoryFileName(jobDetails[JOB_NAME_INDEX]));
        try {
            indexInfo.setFinishTime(Long.parseLong(decodeJobHistoryFileName(jobDetails[FINISH_TIME_INDEX])));
        } catch (NumberFormatException e) {
            LOG.warn("Unable to parse finish time from job history file " + jhFileName + " : " + e);
        }
        try {
            indexInfo.setNumMaps(Integer.parseInt(decodeJobHistoryFileName(jobDetails[NUM_MAPS_INDEX])));
        } catch (NumberFormatException e) {
            LOG.warn("Unable to parse num maps from job history file " + jhFileName + " : " + e);
        }
        try {
            indexInfo.setNumReduces(Integer.parseInt(decodeJobHistoryFileName(jobDetails[NUM_REDUCES_INDEX])));
        } catch (NumberFormatException e) {
            LOG.warn("Unable to parse num reduces from job history file " + jhFileName + " : " + e);
        }
        indexInfo.setJobStatus(decodeJobHistoryFileName(jobDetails[JOB_STATUS_INDEX]));
        indexInfo.setQueueName(decodeJobHistoryFileName(jobDetails[QUEUE_NAME_INDEX]));
        try {
            if (jobDetails.length <= JOB_START_TIME_INDEX) {
                indexInfo.setJobStartTime(indexInfo.getSubmitTime());
            } else {
                indexInfo.setJobStartTime(Long.parseLong(decodeJobHistoryFileName(jobDetails[JOB_START_TIME_INDEX])));
            }
        } catch (NumberFormatException e) {
            LOG.warn("Unable to parse start time from job history file " + jhFileName + " : " + e);
        }
    } catch (IndexOutOfBoundsException e) {
        LOG.warn("Parsing job history file with partial data encoded into name: " + jhFileName);
    }
    return indexInfo;
}

Example 9 with JobIndexInfo

use of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo in project hadoop by apache.

the class TestFileNameIndexUtils method testTrimJobNameEqualsLimitLength.

@Test
public void testTrimJobNameEqualsLimitLength() throws IOException {
    int jobNameTrimLength = 9;
    JobIndexInfo info = new JobIndexInfo();
    JobID oldJobId = JobID.forName(JOB_ID);
    JobId jobId = TypeConverter.toYarn(oldJobId);
    info.setJobId(jobId);
    info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
    info.setUser(USER_NAME);
    info.setJobName(JOB_NAME);
    info.setFinishTime(Long.parseLong(FINISH_TIME));
    info.setNumMaps(Integer.parseInt(NUM_MAPS));
    info.setNumReduces(Integer.parseInt(NUM_REDUCES));
    info.setJobStatus(JOB_STATUS);
    info.setQueueName(QUEUE_NAME);
    info.setJobStartTime(Long.parseLong(JOB_START_TIME));
    String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info, jobNameTrimLength);
    JobIndexInfo parsedInfo = FileNameIndexUtils.getIndexInfo(jobHistoryFile);
    Assert.assertEquals("Job name did not get trimmed correctly", info.getJobName().substring(0, jobNameTrimLength), parsedInfo.getJobName());
}

Example 10 with JobIndexInfo

use of org.apache.hadoop.mapreduce.v2.jobhistory.JobIndexInfo in project hadoop by apache.

the class TestFileNameIndexUtils method testJobNameWithMultibyteChars.

/**
   * Verify the name of jobhistory file is not greater than 255 bytes
   * even if there are some multibyte characters in the job name.
   */
@Test
public void testJobNameWithMultibyteChars() throws IOException {
    JobIndexInfo info = new JobIndexInfo();
    JobID oldJobId = JobID.forName(JOB_ID);
    JobId jobId = TypeConverter.toYarn(oldJobId);
    info.setJobId(jobId);
    info.setSubmitTime(Long.parseLong(SUBMIT_TIME));
    info.setUser(USER_NAME);
    StringBuilder sb = new StringBuilder();
    info.setFinishTime(Long.parseLong(FINISH_TIME));
    info.setNumMaps(Integer.parseInt(NUM_MAPS));
    info.setNumReduces(Integer.parseInt(NUM_REDUCES));
    info.setJobStatus(JOB_STATUS);
    info.setQueueName(QUEUE_NAME);
    info.setJobStartTime(Long.parseLong(JOB_START_TIME));
    // which is encoded into 1 x 3 = 3 characters by URL encode.
    for (int i = 0; i < 100; i++) {
        sb.append('%');
    }
    String longJobName = sb.toString();
    info.setJobName(longJobName);
    String jobHistoryFile = FileNameIndexUtils.getDoneFileName(info, 50);
    Assert.assertTrue(jobHistoryFile.length() <= 255);
    String trimedJobName = jobHistoryFile.split(FileNameIndexUtils.DELIMITER)[// 3 is index of job name
    3];
    // 3 x 16 < 50 < 3 x 17 so the length of trimedJobName should be 48
    Assert.assertEquals(48, trimedJobName.getBytes(UTF_8).length);
    // validate whether trimmedJobName by testing reversibility
    byte[] trimedJobNameInByte = trimedJobName.getBytes(UTF_8);
    String reEncodedTrimedJobName = new String(trimedJobNameInByte, UTF_8);
    Assert.assertArrayEquals(trimedJobNameInByte, reEncodedTrimedJobName.getBytes(UTF_8));
    sb.setLength(0);
    // which is encoded into 2 x 3 = 6 characters by URL encode.
    for (int i = 0; i < 100; i++) {
        // large omega
        sb.append('Ω');
    }
    longJobName = sb.toString();
    info.setJobName(longJobName);
    jobHistoryFile = FileNameIndexUtils.getDoneFileName(info, 27);
    Assert.assertTrue(jobHistoryFile.length() <= 255);
    trimedJobName = jobHistoryFile.split(FileNameIndexUtils.DELIMITER)[// 3 is index of job name
    3];
    // 6 x 4 < 27 < 6 x 5 so the length of trimedJobName should be 24
    Assert.assertEquals(24, trimedJobName.getBytes(UTF_8).length);
    // validate whether trimmedJobName by testing reversibility
    trimedJobNameInByte = trimedJobName.getBytes(UTF_8);
    reEncodedTrimedJobName = new String(trimedJobNameInByte, UTF_8);
    Assert.assertArrayEquals(trimedJobNameInByte, reEncodedTrimedJobName.getBytes(UTF_8));
    sb.setLength(0);
    // which is encoded into 3 x 3 = 9 characters by URL encode.
    for (int i = 0; i < 100; i++) {
        // rightwards arrow
        sb.append('→');
    }
    longJobName = sb.toString();
    info.setJobName(longJobName);
    jobHistoryFile = FileNameIndexUtils.getDoneFileName(info, 40);
    Assert.assertTrue(jobHistoryFile.length() <= 255);
    trimedJobName = jobHistoryFile.split(FileNameIndexUtils.DELIMITER)[// 3 is index of job name
    3];
    // 9 x 4 < 40 < 9 x 5 so the length of trimedJobName should be 36
    Assert.assertEquals(36, trimedJobName.getBytes(UTF_8).length);
    // validate whether trimmedJobName by testing reversibility
    trimedJobNameInByte = trimedJobName.getBytes(UTF_8);
    reEncodedTrimedJobName = new String(trimedJobNameInByte, UTF_8);
    Assert.assertArrayEquals(trimedJobNameInByte, reEncodedTrimedJobName.getBytes(UTF_8));
    sb.setLength(0);
    // which is encoded into 4 x 3 = 12 characters by URL encode.
    for (int i = 0; i < 100; i++) {
        // Mugil cephalus in Kanji.
        sb.append("𩸽");
    }
    longJobName = sb.toString();
    info.setJobName(longJobName);
    jobHistoryFile = FileNameIndexUtils.getDoneFileName(info, 49);
    Assert.assertTrue(jobHistoryFile.length() <= 255);
    trimedJobName = jobHistoryFile.split(FileNameIndexUtils.DELIMITER)[// 3 is index of job name
    3];
    // 12 x 4 < 49 < 12 x 5 so the length of trimedJobName should be 48
    Assert.assertEquals(48, trimedJobName.getBytes(UTF_8).length);
    // validate whether trimmedJobName by testing reversibility
    trimedJobNameInByte = trimedJobName.getBytes(UTF_8);
    reEncodedTrimedJobName = new String(trimedJobNameInByte, UTF_8);
    Assert.assertArrayEquals(trimedJobNameInByte, reEncodedTrimedJobName.getBytes(UTF_8));
    sb.setLength(0);
    // Test for the combination of 1 to 4 bytes UTF-8 characters
    // cat in Kanji (encoded into 3 bytes x 3 characters)
    sb.append('猫').append(// (encoded into 1 byte x 3 characters)
    "[").append(// small lambda (encoded into 2 bytes x 3 characters)
    'λ').append(// (encoded into 1 byte x 3 characters)
    '/').append(// not url-encoded (1 byte x 1 character)
    'A').append(// flying fish in
    "𩹉").append(// dog in Kanji (encoded into 3 bytes x 3 characters)
    '犬');
    longJobName = sb.toString();
    info.setJobName(longJobName);
    jobHistoryFile = FileNameIndexUtils.getDoneFileName(info, 23);
    Assert.assertTrue(jobHistoryFile.length() <= 255);
    trimedJobName = jobHistoryFile.split(FileNameIndexUtils.DELIMITER)[// 3 is index of job name
    3];
    // total size of the first 5 characters = 22
    // 23 < total size of the first 6 characters
    Assert.assertEquals(22, trimedJobName.getBytes(UTF_8).length);
    // validate whether trimmedJobName by testing reversibility
    trimedJobNameInByte = trimedJobName.getBytes(UTF_8);
    reEncodedTrimedJobName = new String(trimedJobNameInByte, UTF_8);
    Assert.assertArrayEquals(trimedJobNameInByte, reEncodedTrimedJobName.getBytes(UTF_8));
}