Examples with TaskAttemptReport org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport used on opensource projects

Example 1 with TaskAttemptReport

use of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport in project hadoop by apache.

the class ClientServiceDelegate method getLogFilePath.

public LogParams getLogFilePath(JobID oldJobID, TaskAttemptID oldTaskAttemptID) throws IOException {
    org.apache.hadoop.mapreduce.v2.api.records.JobId jobId = TypeConverter.toYarn(oldJobID);
    GetJobReportRequest request = recordFactory.newRecordInstance(GetJobReportRequest.class);
    request.setJobId(jobId);
    JobReport report = ((GetJobReportResponse) invoke("getJobReport", GetJobReportRequest.class, request)).getJobReport();
    if (EnumSet.of(JobState.SUCCEEDED, JobState.FAILED, JobState.KILLED, JobState.ERROR).contains(report.getJobState())) {
        if (oldTaskAttemptID != null) {
            GetTaskAttemptReportRequest taRequest = recordFactory.newRecordInstance(GetTaskAttemptReportRequest.class);
            taRequest.setTaskAttemptId(TypeConverter.toYarn(oldTaskAttemptID));
            TaskAttemptReport taReport = ((GetTaskAttemptReportResponse) invoke("getTaskAttemptReport", GetTaskAttemptReportRequest.class, taRequest)).getTaskAttemptReport();
            if (taReport.getContainerId() == null || taReport.getNodeManagerHost() == null) {
                throw new IOException("Unable to get log information for task: " + oldTaskAttemptID);
            }
            return new LogParams(taReport.getContainerId().toString(), taReport.getContainerId().getApplicationAttemptId().getApplicationId().toString(), NodeId.newInstance(taReport.getNodeManagerHost(), taReport.getNodeManagerPort()).toString(), report.getUser());
        } else {
            if (report.getAMInfos() == null || report.getAMInfos().size() == 0) {
                throw new IOException("Unable to get log information for job: " + oldJobID);
            }
            AMInfo amInfo = report.getAMInfos().get(report.getAMInfos().size() - 1);
            return new LogParams(amInfo.getContainerId().toString(), amInfo.getAppAttemptId().getApplicationId().toString(), NodeId.newInstance(amInfo.getNodeManagerHost(), amInfo.getNodeManagerPort()).toString(), report.getUser());
        }
    } else {
        throw new IOException("Cannot get log path for a in-progress job");
    }
}

Example 2 with TaskAttemptReport

use of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport in project hadoop by apache.

the class TestJobHistoryEntities method testCompletedTaskAttempt.

@Test(timeout = 10000)
public void testCompletedTaskAttempt() throws Exception {
    HistoryFileInfo info = mock(HistoryFileInfo.class);
    when(info.getConfFile()).thenReturn(fullConfPath);
    completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager);
    TaskId mt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
    TaskId rt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
    TaskAttemptId mta1Id = MRBuilderUtils.newTaskAttemptId(mt1Id, 0);
    TaskAttemptId rta1Id = MRBuilderUtils.newTaskAttemptId(rt1Id, 0);
    Task mt1 = completedJob.getTask(mt1Id);
    Task rt1 = completedJob.getTask(rt1Id);
    TaskAttempt mta1 = mt1.getAttempt(mta1Id);
    assertEquals(TaskAttemptState.SUCCEEDED, mta1.getState());
    assertEquals("localhost:45454", mta1.getAssignedContainerMgrAddress());
    assertEquals("localhost:9999", mta1.getNodeHttpAddress());
    TaskAttemptReport mta1Report = mta1.getReport();
    assertEquals(TaskAttemptState.SUCCEEDED, mta1Report.getTaskAttemptState());
    assertEquals("localhost", mta1Report.getNodeManagerHost());
    assertEquals(45454, mta1Report.getNodeManagerPort());
    assertEquals(9999, mta1Report.getNodeManagerHttpPort());
    TaskAttempt rta1 = rt1.getAttempt(rta1Id);
    assertEquals(TaskAttemptState.SUCCEEDED, rta1.getState());
    assertEquals("localhost:45454", rta1.getAssignedContainerMgrAddress());
    assertEquals("localhost:9999", rta1.getNodeHttpAddress());
    TaskAttemptReport rta1Report = rta1.getReport();
    assertEquals(TaskAttemptState.SUCCEEDED, rta1Report.getTaskAttemptState());
    assertEquals("localhost", rta1Report.getNodeManagerHost());
    assertEquals(45454, rta1Report.getNodeManagerPort());
    assertEquals(9999, rta1Report.getNodeManagerHttpPort());
}

Example 3 with TaskAttemptReport

use of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport in project hadoop by apache.

the class TestBlocks method testAttemptsBlock.

/**
   * test AttemptsBlock's rendering.
   */
@Test
public void testAttemptsBlock() {
    AppContext ctx = mock(AppContext.class);
    AppForTest app = new AppForTest(ctx);
    Task task = getTask(0);
    Map<TaskAttemptId, TaskAttempt> attempts = new HashMap<TaskAttemptId, TaskAttempt>();
    TaskAttempt attempt = mock(TaskAttempt.class);
    TaskAttemptId taId = new TaskAttemptIdPBImpl();
    taId.setId(0);
    taId.setTaskId(task.getID());
    when(attempt.getID()).thenReturn(taId);
    when(attempt.getNodeHttpAddress()).thenReturn("Node address");
    ApplicationId appId = ApplicationIdPBImpl.newInstance(0, 5);
    ApplicationAttemptId appAttemptId = ApplicationAttemptIdPBImpl.newInstance(appId, 1);
    ContainerId containerId = ContainerIdPBImpl.newContainerId(appAttemptId, 1);
    when(attempt.getAssignedContainerID()).thenReturn(containerId);
    when(attempt.getAssignedContainerMgrAddress()).thenReturn("assignedContainerMgrAddress");
    when(attempt.getNodeRackName()).thenReturn("nodeRackName");
    final long taStartTime = 100002L;
    final long taFinishTime = 100012L;
    final long taShuffleFinishTime = 100010L;
    final long taSortFinishTime = 100011L;
    final TaskAttemptState taState = TaskAttemptState.SUCCEEDED;
    when(attempt.getLaunchTime()).thenReturn(taStartTime);
    when(attempt.getFinishTime()).thenReturn(taFinishTime);
    when(attempt.getShuffleFinishTime()).thenReturn(taShuffleFinishTime);
    when(attempt.getSortFinishTime()).thenReturn(taSortFinishTime);
    when(attempt.getState()).thenReturn(taState);
    TaskAttemptReport taReport = mock(TaskAttemptReport.class);
    when(taReport.getStartTime()).thenReturn(taStartTime);
    when(taReport.getFinishTime()).thenReturn(taFinishTime);
    when(taReport.getShuffleFinishTime()).thenReturn(taShuffleFinishTime);
    when(taReport.getSortFinishTime()).thenReturn(taSortFinishTime);
    when(taReport.getContainerId()).thenReturn(containerId);
    when(taReport.getProgress()).thenReturn(1.0f);
    when(taReport.getStateString()).thenReturn("Processed 128/128 records <p> \n");
    when(taReport.getTaskAttemptState()).thenReturn(taState);
    when(taReport.getDiagnosticInfo()).thenReturn("");
    when(attempt.getReport()).thenReturn(taReport);
    attempts.put(taId, attempt);
    when(task.getAttempts()).thenReturn(attempts);
    app.setTask(task);
    Job job = mock(Job.class);
    when(job.getUserName()).thenReturn("User");
    app.setJob(job);
    AttemptsBlockForTest block = new AttemptsBlockForTest(app);
    block.addParameter(AMParams.TASK_TYPE, "r");
    PrintWriter pWriter = new PrintWriter(data);
    Block html = new BlockForTest(new HtmlBlockForTest(), pWriter, 0, false);
    block.render(html);
    pWriter.flush();
    // should be printed information about attempts
    assertTrue(data.toString().contains("attempt_0_0001_r_000000_0"));
    assertTrue(data.toString().contains("SUCCEEDED"));
    assertFalse(data.toString().contains("Processed 128/128 records <p> \n"));
    assertTrue(data.toString().contains("Processed 128\\/128 records &lt;p&gt; \\n"));
    assertTrue(data.toString().contains("_0005_01_000001:attempt_0_0001_r_000000_0:User:"));
    assertTrue(data.toString().contains("100002"));
    assertTrue(data.toString().contains("100010"));
    assertTrue(data.toString().contains("100011"));
    assertTrue(data.toString().contains("100012"));
}

Example 4 with TaskAttemptReport

use of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport in project hadoop by apache.

the class MRApp method waitForState.

public void waitForState(TaskAttempt attempt, TaskAttemptState finalState) throws Exception {
    int timeoutSecs = 0;
    TaskAttemptReport report = attempt.getReport();
    while (!finalState.equals(report.getTaskAttemptState()) && timeoutSecs++ < 20) {
        System.out.println("TaskAttempt State is : " + report.getTaskAttemptState() + " Waiting for state : " + finalState + "   progress : " + report.getProgress());
        report = attempt.getReport();
        Thread.sleep(500);
    }
    System.out.println("TaskAttempt State is : " + report.getTaskAttemptState());
    Assert.assertEquals("TaskAttempt state is not correct (timedout)", finalState, report.getTaskAttemptState());
}

Example 5 with TaskAttemptReport

use of org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptReport in project hadoop by apache.

the class MRApp method verifyCompleted.

public void verifyCompleted() {
    for (Job job : getContext().getAllJobs().values()) {
        JobReport jobReport = job.getReport();
        System.out.println("Job start time :" + jobReport.getStartTime());
        System.out.println("Job finish time :" + jobReport.getFinishTime());
        Assert.assertTrue("Job start time is not less than finish time", jobReport.getStartTime() <= jobReport.getFinishTime());
        Assert.assertTrue("Job finish time is in future", jobReport.getFinishTime() <= System.currentTimeMillis());
        for (Task task : job.getTasks().values()) {
            TaskReport taskReport = task.getReport();
            System.out.println("Task start time : " + taskReport.getStartTime());
            System.out.println("Task finish time : " + taskReport.getFinishTime());
            Assert.assertTrue("Task start time is not less than finish time", taskReport.getStartTime() <= taskReport.getFinishTime());
            for (TaskAttempt attempt : task.getAttempts().values()) {
                TaskAttemptReport attemptReport = attempt.getReport();
                Assert.assertTrue("Attempt start time is not less than finish time", attemptReport.getStartTime() <= attemptReport.getFinishTime());
            }
        }
    }
}