Examples with Job org.apache.hadoop.mapreduce.v2.app.job.Job used on opensource projects

Example 56 with Job

use of org.apache.hadoop.mapreduce.v2.app.job.Job in project hadoop by apache.

the class MockHistoryJobs method split.

private static JobsPair split(Map<JobId, Job> mocked) throws IOException {
    JobsPair ret = new JobsPair();
    ret.full = Maps.newHashMap();
    ret.partial = Maps.newHashMap();
    for (Map.Entry<JobId, Job> entry : mocked.entrySet()) {
        JobId id = entry.getKey();
        Job j = entry.getValue();
        MockCompletedJob mockJob = new MockCompletedJob(j);
        // use MockCompletedJob to set everything below to make sure
        // consistent with what history server would do
        ret.full.put(id, mockJob);
        JobReport report = mockJob.getReport();
        JobIndexInfo info = new JobIndexInfo(report.getStartTime(), report.getFinishTime(), mockJob.getUserName(), mockJob.getName(), id, mockJob.getCompletedMaps(), mockJob.getCompletedReduces(), String.valueOf(mockJob.getState()));
        info.setJobStartTime(report.getStartTime());
        info.setQueueName(mockJob.getQueueName());
        ret.partial.put(id, new PartialJob(info, id));
    }
    return ret;
}

Example 57 with Job

use of org.apache.hadoop.mapreduce.v2.app.job.Job in project hadoop by apache.

the class TestHistoryFileManager method testHistoryFileInfoSummaryFileNotExist.

@Test
public void testHistoryFileInfoSummaryFileNotExist() throws Exception {
    HistoryFileManagerTest hmTest = new HistoryFileManagerTest();
    String job = "job_1410889000000_123456";
    Path summaryFile = new Path(job + ".summary");
    JobIndexInfo jobIndexInfo = new JobIndexInfo();
    jobIndexInfo.setJobId(TypeConverter.toYarn(JobID.forName(job)));
    Configuration conf = dfsCluster.getConfiguration(0);
    conf.set(JHAdminConfig.MR_HISTORY_DONE_DIR, "/" + UUID.randomUUID());
    conf.set(JHAdminConfig.MR_HISTORY_INTERMEDIATE_DONE_DIR, "/" + UUID.randomUUID());
    hmTest.serviceInit(conf);
    HistoryFileInfo info = hmTest.getHistoryFileInfo(null, null, summaryFile, jobIndexInfo, false);
    info.moveToDone();
    Assert.assertFalse(info.didMoveFail());
}

Example 58 with Job

use of org.apache.hadoop.mapreduce.v2.app.job.Job in project hadoop by apache.

the class TestBlocks method testHsJobsBlock.

/**
   * test HsJobsBlock's rendering.
   */
@Test
public void testHsJobsBlock() {
    AppContext ctx = mock(AppContext.class);
    Map<JobId, Job> jobs = new HashMap<JobId, Job>();
    Job job = getJob();
    jobs.put(job.getID(), job);
    when(ctx.getAllJobs()).thenReturn(jobs);
    HsJobsBlock block = new HsJobsBlockForTest(ctx);
    PrintWriter pWriter = new PrintWriter(data);
    Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);
    block.render(html);
    pWriter.flush();
    assertTrue(data.toString().contains("JobName"));
    assertTrue(data.toString().contains("UserName"));
    assertTrue(data.toString().contains("QueueName"));
    assertTrue(data.toString().contains("SUCCEEDED"));
}

Example 59 with Job

use of org.apache.hadoop.mapreduce.v2.app.job.Job in project hadoop by apache.

the class TestBlocks method testAttemptsBlock.

/**
   * test AttemptsBlock's rendering.
   */
@Test
public void testAttemptsBlock() {
    AppContext ctx = mock(AppContext.class);
    AppForTest app = new AppForTest(ctx);
    Task task = getTask(0);
    Map<TaskAttemptId, TaskAttempt> attempts = new HashMap<TaskAttemptId, TaskAttempt>();
    TaskAttempt attempt = mock(TaskAttempt.class);
    TaskAttemptId taId = new TaskAttemptIdPBImpl();
    taId.setId(0);
    taId.setTaskId(task.getID());
    when(attempt.getID()).thenReturn(taId);
    when(attempt.getNodeHttpAddress()).thenReturn("Node address");
    ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 5);
    ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.newInstance(appId, 1);
    ContainerId containerId = ContainerIdPBImpl.newContainerId(appAttemptId, 1);
    when(attempt.getAssignedContainerID()).thenReturn(containerId);
    when(attempt.getAssignedContainerMgrAddress()).thenReturn("assignedContainerMgrAddress");
    when(attempt.getNodeRackName()).thenReturn("nodeRackName");
    final long taStartTime = 100002L;
    final long taFinishTime = 100012L;
    final long taShuffleFinishTime = 100010L;
    final long taSortFinishTime = 100011L;
    final TaskAttemptState taState = TaskAttemptState.SUCCEEDED;
    when(attempt.getLaunchTime()).thenReturn(taStartTime);
    when(attempt.getFinishTime()).thenReturn(taFinishTime);
    when(attempt.getShuffleFinishTime()).thenReturn(taShuffleFinishTime);
    when(attempt.getSortFinishTime()).thenReturn(taSortFinishTime);
    when(attempt.getState()).thenReturn(taState);
    TaskAttemptReport taReport = mock(TaskAttemptReport.class);
    when(taReport.getStartTime()).thenReturn(taStartTime);
    when(taReport.getFinishTime()).thenReturn(taFinishTime);
    when(taReport.getShuffleFinishTime()).thenReturn(taShuffleFinishTime);
    when(taReport.getSortFinishTime()).thenReturn(taSortFinishTime);
    when(taReport.getContainerId()).thenReturn(containerId);
    when(taReport.getProgress()).thenReturn(1.0f);
    when(taReport.getStateString()).thenReturn("Processed 128/128 records <p> \n");
    when(taReport.getTaskAttemptState()).thenReturn(taState);
    when(taReport.getDiagnosticInfo()).thenReturn("");
    when(attempt.getReport()).thenReturn(taReport);
    attempts.put(taId, attempt);
    when(task.getAttempts()).thenReturn(attempts);
    app.setTask(task);
    Job job = mock(Job.class);
    when(job.getUserName()).thenReturn("User");
    app.setJob(job);
    AttemptsBlockForTest block = new AttemptsBlockForTest(app);
    block.addParameter(AMParams.TASK_TYPE, "r");
    PrintWriter pWriter = new PrintWriter(data);
    Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);
    block.render(html);
    pWriter.flush();
    // should be printed information about attempts
    assertTrue(data.toString().contains("attempt_0_0001_r_000000_0"));
    assertTrue(data.toString().contains("SUCCEEDED"));
    assertFalse(data.toString().contains("Processed 128/128 records <p> \n"));
    assertTrue(data.toString().contains("Processed 128\\/128 records &lt;p&gt; \\n"));
    assertTrue(data.toString().contains("_0005_01_000001:attempt_0_0001_r_000000_0:User:"));
    assertTrue(data.toString().contains("100002"));
    assertTrue(data.toString().contains("100010"));
    assertTrue(data.toString().contains("100011"));
    assertTrue(data.toString().contains("100012"));
}

Example 60 with Job

use of org.apache.hadoop.mapreduce.v2.app.job.Job in project hadoop by apache.

the class TestBlocks method testPullTaskLink.

@Test
public void testPullTaskLink() {
    Task task = getTask(0);
    String taskId = task.getID().toString();
    Assert.assertEquals("pull links doesn't work correctly", "Task failed <a href=\"/jobhistory/task/" + taskId + "\">" + taskId + "</a>", HsJobBlock.addTaskLinks("Task failed " + taskId));
    Assert.assertEquals("pull links doesn't work correctly", "Task failed <a href=\"/jobhistory/task/" + taskId + "\">" + taskId + "</a>\n Job failed as tasks failed. failedMaps:1 failedReduces:0", HsJobBlock.addTaskLinks("Task failed " + taskId + "\n " + "Job failed as tasks failed. failedMaps:1 failedReduces:0"));
}