Examples with JobReport org.apache.hadoop.mapreduce.v2.api.records.JobReport used on opensource projects

Example 21 with JobReport

use of org.apache.hadoop.mapreduce.v2.api.records.JobReport in project hadoop by apache.

the class CachedHistoryStorage method getPartialJobs.

public static JobsInfo getPartialJobs(Collection<Job> jobs, Long offset, Long count, String user, String queue, Long sBegin, Long sEnd, Long fBegin, Long fEnd, JobState jobState) {
    JobsInfo allJobs = new JobsInfo();
    if (sBegin == null || sBegin < 0)
        sBegin = 0l;
    if (sEnd == null)
        sEnd = Long.MAX_VALUE;
    if (fBegin == null || fBegin < 0)
        fBegin = 0l;
    if (fEnd == null)
        fEnd = Long.MAX_VALUE;
    if (offset == null || offset < 0)
        offset = 0l;
    if (count == null)
        count = Long.MAX_VALUE;
    if (offset > jobs.size()) {
        return allJobs;
    }
    long at = 0;
    long end = offset + count - 1;
    if (end < 0) {
        // due to overflow
        end = Long.MAX_VALUE;
    }
    for (Job job : jobs) {
        if (at > end) {
            break;
        }
        // can't really validate queue is a valid one since queues could change
        if (queue != null && !queue.isEmpty()) {
            if (!job.getQueueName().equals(queue)) {
                continue;
            }
        }
        if (user != null && !user.isEmpty()) {
            if (!job.getUserName().equals(user)) {
                continue;
            }
        }
        JobReport report = job.getReport();
        if (report.getStartTime() < sBegin || report.getStartTime() > sEnd) {
            continue;
        }
        if (report.getFinishTime() < fBegin || report.getFinishTime() > fEnd) {
            continue;
        }
        if (jobState != null && jobState != report.getJobState()) {
            continue;
        }
        at++;
        if ((at - 1) < offset) {
            continue;
        }
        JobInfo jobInfo = new JobInfo(job);
        allJobs.add(jobInfo);
    }
    return allJobs;
}

Example 22 with JobReport

use of org.apache.hadoop.mapreduce.v2.api.records.JobReport in project hadoop by apache.

the class TestBlocks method getJob.

private Job getJob() {
    Job job = mock(Job.class);
    JobId jobId = new JobIdPBImpl();
    ApplicationId appId = ApplicationIdPBImpl.newInstance(System.currentTimeMillis(), 4);
    jobId.setAppId(appId);
    jobId.setId(1);
    when(job.getID()).thenReturn(jobId);
    JobReport report = mock(JobReport.class);
    when(report.getStartTime()).thenReturn(100010L);
    when(report.getFinishTime()).thenReturn(100015L);
    when(job.getReport()).thenReturn(report);
    when(job.getName()).thenReturn("JobName");
    when(job.getUserName()).thenReturn("UserName");
    when(job.getQueueName()).thenReturn("QueueName");
    when(job.getState()).thenReturn(JobState.SUCCEEDED);
    when(job.getTotalMaps()).thenReturn(3);
    when(job.getCompletedMaps()).thenReturn(2);
    when(job.getTotalReduces()).thenReturn(2);
    when(job.getCompletedReduces()).thenReturn(1);
    when(job.getCompletedReduces()).thenReturn(1);
    return job;
}

Example 23 with JobReport

use of org.apache.hadoop.mapreduce.v2.api.records.JobReport in project hadoop by apache.

the class TestJobHistoryEntities method testCompletedJob.

/* Verify some expected values based on the history file */
@Test(timeout = 100000)
public void testCompletedJob() throws Exception {
    HistoryFileInfo info = mock(HistoryFileInfo.class);
    when(info.getConfFile()).thenReturn(fullConfPath);
    when(info.getHistoryFile()).thenReturn(fullHistoryPath);
    //Re-initialize to verify the delayed load.
    completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager);
    //Verify tasks loaded based on loadTask parameter.
    assertEquals(loadTasks, completedJob.tasksLoaded.get());
    assertEquals(1, completedJob.getAMInfos().size());
    assertEquals(10, completedJob.getCompletedMaps());
    assertEquals(1, completedJob.getCompletedReduces());
    assertEquals(12, completedJob.getTasks().size());
    //Verify tasks loaded at this point.
    assertEquals(true, completedJob.tasksLoaded.get());
    assertEquals(10, completedJob.getTasks(TaskType.MAP).size());
    assertEquals(2, completedJob.getTasks(TaskType.REDUCE).size());
    assertEquals("user", completedJob.getUserName());
    assertEquals(JobState.SUCCEEDED, completedJob.getState());
    JobReport jobReport = completedJob.getReport();
    assertEquals("user", jobReport.getUser());
    assertEquals(JobState.SUCCEEDED, jobReport.getJobState());
    assertEquals(fullHistoryPath.toString(), jobReport.getHistoryFile());
}

Example 24 with JobReport

use of org.apache.hadoop.mapreduce.v2.api.records.JobReport in project hadoop by apache.

the class VerifyJobsUtils method verifyHsJobGeneric.

public static void verifyHsJobGeneric(Job job, String id, String user, String name, String state, String queue, long startTime, long finishTime, int mapsTotal, int mapsCompleted, int reducesTotal, int reducesCompleted) {
    JobReport report = job.getReport();
    WebServicesTestUtils.checkStringMatch("id", MRApps.toString(job.getID()), id);
    WebServicesTestUtils.checkStringMatch("user", job.getUserName().toString(), user);
    WebServicesTestUtils.checkStringMatch("name", job.getName(), name);
    WebServicesTestUtils.checkStringMatch("state", job.getState().toString(), state);
    WebServicesTestUtils.checkStringMatch("queue", job.getQueueName(), queue);
    assertEquals("startTime incorrect", report.getStartTime(), startTime);
    assertEquals("finishTime incorrect", report.getFinishTime(), finishTime);
    assertEquals("mapsTotal incorrect", job.getTotalMaps(), mapsTotal);
    assertEquals("mapsCompleted incorrect", job.getCompletedMaps(), mapsCompleted);
    assertEquals("reducesTotal incorrect", job.getTotalReduces(), reducesTotal);
    assertEquals("reducesCompleted incorrect", job.getCompletedReduces(), reducesCompleted);
}

Example 25 with JobReport

use of org.apache.hadoop.mapreduce.v2.api.records.JobReport in project hadoop by apache.

the class TestJobInfo method testAverageReduceTime.

@Test
public void testAverageReduceTime() {
    Job job = mock(CompletedJob.class);
    final Task task1 = mock(Task.class);
    final Task task2 = mock(Task.class);
    JobId jobId = MRBuilderUtils.newJobId(1L, 1, 1);
    final TaskId taskId1 = MRBuilderUtils.newTaskId(jobId, 1, TaskType.REDUCE);
    final TaskId taskId2 = MRBuilderUtils.newTaskId(jobId, 2, TaskType.REDUCE);
    final TaskAttemptId taskAttemptId1 = MRBuilderUtils.newTaskAttemptId(taskId1, 1);
    final TaskAttemptId taskAttemptId2 = MRBuilderUtils.newTaskAttemptId(taskId2, 2);
    final TaskAttempt taskAttempt1 = mock(TaskAttempt.class);
    final TaskAttempt taskAttempt2 = mock(TaskAttempt.class);
    JobReport jobReport = mock(JobReport.class);
    when(taskAttempt1.getState()).thenReturn(TaskAttemptState.SUCCEEDED);
    when(taskAttempt1.getLaunchTime()).thenReturn(0L);
    when(taskAttempt1.getShuffleFinishTime()).thenReturn(4L);
    when(taskAttempt1.getSortFinishTime()).thenReturn(6L);
    when(taskAttempt1.getFinishTime()).thenReturn(8L);
    when(taskAttempt2.getState()).thenReturn(TaskAttemptState.SUCCEEDED);
    when(taskAttempt2.getLaunchTime()).thenReturn(5L);
    when(taskAttempt2.getShuffleFinishTime()).thenReturn(10L);
    when(taskAttempt2.getSortFinishTime()).thenReturn(22L);
    when(taskAttempt2.getFinishTime()).thenReturn(42L);
    when(task1.getType()).thenReturn(TaskType.REDUCE);
    when(task2.getType()).thenReturn(TaskType.REDUCE);
    when(task1.getAttempts()).thenReturn(new HashMap<TaskAttemptId, TaskAttempt>() {

        {
            put(taskAttemptId1, taskAttempt1);
        }
    });
    when(task2.getAttempts()).thenReturn(new HashMap<TaskAttemptId, TaskAttempt>() {

        {
            put(taskAttemptId2, taskAttempt2);
        }
    });
    when(job.getTasks()).thenReturn(new HashMap<TaskId, Task>() {

        {
            put(taskId1, task1);
            put(taskId2, task2);
        }
    });
    when(job.getID()).thenReturn(jobId);
    when(job.getReport()).thenReturn(jobReport);
    when(job.getName()).thenReturn("TestJobInfo");
    when(job.getState()).thenReturn(JobState.SUCCEEDED);
    JobInfo jobInfo = new JobInfo(job);
    Assert.assertEquals(11L, jobInfo.getAvgReduceTime().longValue());
}