Examples with JobId org.apache.hadoop.mapreduce.v2.api.records.JobId used on opensource projects

Example 46 with JobId

use of org.apache.hadoop.mapreduce.v2.api.records.JobId in project hadoop by apache.

the class TestJobHistoryEntities method testCompletedTaskAttempt.

@Test(timeout = 10000)
public void testCompletedTaskAttempt() throws Exception {
    HistoryFileInfo info = mock(HistoryFileInfo.class);
    when(info.getConfFile()).thenReturn(fullConfPath);
    completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager);
    TaskId mt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
    TaskId rt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
    TaskAttemptId mta1Id = MRBuilderUtils.newTaskAttemptId(mt1Id, 0);
    TaskAttemptId rta1Id = MRBuilderUtils.newTaskAttemptId(rt1Id, 0);
    Task mt1 = completedJob.getTask(mt1Id);
    Task rt1 = completedJob.getTask(rt1Id);
    TaskAttempt mta1 = mt1.getAttempt(mta1Id);
    assertEquals(TaskAttemptState.SUCCEEDED, mta1.getState());
    assertEquals("localhost:45454", mta1.getAssignedContainerMgrAddress());
    assertEquals("localhost:9999", mta1.getNodeHttpAddress());
    TaskAttemptReport mta1Report = mta1.getReport();
    assertEquals(TaskAttemptState.SUCCEEDED, mta1Report.getTaskAttemptState());
    assertEquals("localhost", mta1Report.getNodeManagerHost());
    assertEquals(45454, mta1Report.getNodeManagerPort());
    assertEquals(9999, mta1Report.getNodeManagerHttpPort());
    TaskAttempt rta1 = rt1.getAttempt(rta1Id);
    assertEquals(TaskAttemptState.SUCCEEDED, rta1.getState());
    assertEquals("localhost:45454", rta1.getAssignedContainerMgrAddress());
    assertEquals("localhost:9999", rta1.getNodeHttpAddress());
    TaskAttemptReport rta1Report = rta1.getReport();
    assertEquals(TaskAttemptState.SUCCEEDED, rta1Report.getTaskAttemptState());
    assertEquals("localhost", rta1Report.getNodeManagerHost());
    assertEquals(45454, rta1Report.getNodeManagerPort());
    assertEquals(9999, rta1Report.getNodeManagerHttpPort());
}

Example 47 with JobId

use of org.apache.hadoop.mapreduce.v2.api.records.JobId in project hadoop by apache.

the class TestJobHistoryEntities method testCopmletedJobReportWithZeroTasks.

@Test(timeout = 100000)
public void testCopmletedJobReportWithZeroTasks() throws Exception {
    HistoryFileInfo info = mock(HistoryFileInfo.class);
    when(info.getConfFile()).thenReturn(fullConfPath);
    when(info.getHistoryFile()).thenReturn(fullHistoryPathZeroReduces);
    completedJob = new CompletedJob(conf, jobId, fullHistoryPathZeroReduces, loadTasks, "user", info, jobAclsManager);
    JobReport jobReport = completedJob.getReport();
    // Make sure that the number reduces (completed and total) are equal to zero.
    assertEquals(0, completedJob.getTotalReduces());
    assertEquals(0, completedJob.getCompletedReduces());
    // Verify that the reduce progress is 1.0 (not NaN)
    assertEquals(1.0, jobReport.getReduceProgress(), 0.001);
    assertEquals(fullHistoryPathZeroReduces.toString(), jobReport.getHistoryFile());
}

Example 48 with JobId

use of org.apache.hadoop.mapreduce.v2.api.records.JobId in project hadoop by apache.

the class TestJobHistoryEntities method testCompletedTask.

@Test(timeout = 10000)
public void testCompletedTask() throws Exception {
    HistoryFileInfo info = mock(HistoryFileInfo.class);
    when(info.getConfFile()).thenReturn(fullConfPath);
    completedJob = new CompletedJob(conf, jobId, fullHistoryPath, loadTasks, "user", info, jobAclsManager);
    TaskId mt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.MAP);
    TaskId rt1Id = MRBuilderUtils.newTaskId(jobId, 0, TaskType.REDUCE);
    Map<TaskId, Task> mapTasks = completedJob.getTasks(TaskType.MAP);
    Map<TaskId, Task> reduceTasks = completedJob.getTasks(TaskType.REDUCE);
    assertEquals(10, mapTasks.size());
    assertEquals(2, reduceTasks.size());
    Task mt1 = mapTasks.get(mt1Id);
    assertEquals(1, mt1.getAttempts().size());
    assertEquals(TaskState.SUCCEEDED, mt1.getState());
    TaskReport mt1Report = mt1.getReport();
    assertEquals(TaskState.SUCCEEDED, mt1Report.getTaskState());
    assertEquals(mt1Id, mt1Report.getTaskId());
    Task rt1 = reduceTasks.get(rt1Id);
    assertEquals(1, rt1.getAttempts().size());
    assertEquals(TaskState.SUCCEEDED, rt1.getState());
    TaskReport rt1Report = rt1.getReport();
    assertEquals(TaskState.SUCCEEDED, rt1Report.getTaskState());
    assertEquals(rt1Id, rt1Report.getTaskId());
}

Example 49 with JobId

use of org.apache.hadoop.mapreduce.v2.api.records.JobId in project hadoop by apache.

the class HsJobBlock method render.

/*
   * (non-Javadoc)
   * @see org.apache.hadoop.yarn.webapp.view.HtmlBlock#render(org.apache.hadoop.yarn.webapp.view.HtmlBlock.Block)
   */
@Override
protected void render(Block html) {
    String jid = $(JOB_ID);
    if (jid.isEmpty()) {
        html.p()._("Sorry, can't do anything without a JobID.")._();
        return;
    }
    JobId jobID = MRApps.toJobID(jid);
    Job j = appContext.getJob(jobID);
    if (j == null) {
        html.p()._("Sorry, ", jid, " not found.")._();
        return;
    }
    if (j instanceof UnparsedJob) {
        final int taskCount = j.getTotalMaps() + j.getTotalReduces();
        UnparsedJob oversizedJob = (UnparsedJob) j;
        html.p()._("The job has a total of " + taskCount + " tasks. ")._("Any job larger than " + oversizedJob.getMaxTasksAllowed() + " will not be loaded.")._();
        html.p()._("You can either use the CLI tool: 'mapred job -history'" + " to view large jobs or adjust the property " + JHAdminConfig.MR_HS_LOADED_JOBS_TASKS_MAX + ".")._();
        return;
    }
    List<AMInfo> amInfos = j.getAMInfos();
    JobInfo job = new JobInfo(j);
    ResponseInfo infoBlock = info("Job Overview")._("Job Name:", job.getName())._("User Name:", job.getUserName())._("Queue:", job.getQueueName())._("State:", job.getState())._("Uberized:", job.isUber())._("Submitted:", new Date(job.getSubmitTime()))._("Started:", job.getStartTimeStr())._("Finished:", new Date(job.getFinishTime()))._("Elapsed:", StringUtils.formatTime(Times.elapsed(job.getStartTime(), job.getFinishTime(), false)));
    String amString = amInfos.size() == 1 ? "ApplicationMaster" : "ApplicationMasters";
    // todo - switch to use JobInfo
    List<String> diagnostics = j.getDiagnostics();
    if (diagnostics != null && !diagnostics.isEmpty()) {
        StringBuffer b = new StringBuffer();
        for (String diag : diagnostics) {
            b.append(addTaskLinks(diag));
        }
        infoBlock._r("Diagnostics:", b.toString());
    }
    if (job.getNumMaps() > 0) {
        infoBlock._("Average Map Time", StringUtils.formatTime(job.getAvgMapTime()));
    }
    if (job.getNumReduces() > 0) {
        infoBlock._("Average Shuffle Time", StringUtils.formatTime(job.getAvgShuffleTime()));
        infoBlock._("Average Merge Time", StringUtils.formatTime(job.getAvgMergeTime()));
        infoBlock._("Average Reduce Time", StringUtils.formatTime(job.getAvgReduceTime()));
    }
    for (ConfEntryInfo entry : job.getAcls()) {
        infoBlock._("ACL " + entry.getName() + ":", entry.getValue());
    }
    DIV<Hamlet> div = html._(InfoBlock.class).div(_INFO_WRAP);
    // MRAppMasters Table
    TABLE<DIV<Hamlet>> table = div.table("#job");
    table.tr().th(amString)._().tr().th(_TH, "Attempt Number").th(_TH, "Start Time").th(_TH, "Node").th(_TH, "Logs")._();
    boolean odd = false;
    for (AMInfo amInfo : amInfos) {
        AMAttemptInfo attempt = new AMAttemptInfo(amInfo, job.getId(), job.getUserName(), "", "");
        table.tr((odd = !odd) ? _ODD : _EVEN).td(String.valueOf(attempt.getAttemptId())).td(new Date(attempt.getStartTime()).toString()).td().a(".nodelink", url(MRWebAppUtil.getYARNWebappScheme(), attempt.getNodeHttpAddress()), attempt.getNodeHttpAddress())._().td().a(".logslink", url(attempt.getLogsLink()), "logs")._()._();
    }
    table._();
    div._();
    html.div(_INFO_WRAP).table("#job").tr().th(_TH, "Task Type").th(_TH, "Total").th(_TH, "Complete")._().tr(_ODD).th().a(url("tasks", jid, "m"), "Map")._().td(String.valueOf(String.valueOf(job.getMapsTotal()))).td(String.valueOf(String.valueOf(job.getMapsCompleted())))._().tr(_EVEN).th().a(url("tasks", jid, "r"), "Reduce")._().td(String.valueOf(String.valueOf(job.getReducesTotal()))).td(String.valueOf(String.valueOf(job.getReducesCompleted())))._()._().table("#job").tr().th(_TH, "Attempt Type").th(_TH, "Failed").th(_TH, "Killed").th(_TH, "Successful")._().tr(_ODD).th("Maps").td().a(url("attempts", jid, "m", TaskAttemptStateUI.FAILED.toString()), String.valueOf(job.getFailedMapAttempts()))._().td().a(url("attempts", jid, "m", TaskAttemptStateUI.KILLED.toString()), String.valueOf(job.getKilledMapAttempts()))._().td().a(url("attempts", jid, "m", TaskAttemptStateUI.SUCCESSFUL.toString()), String.valueOf(job.getSuccessfulMapAttempts()))._()._().tr(_EVEN).th("Reduces").td().a(url("attempts", jid, "r", TaskAttemptStateUI.FAILED.toString()), String.valueOf(job.getFailedReduceAttempts()))._().td().a(url("attempts", jid, "r", TaskAttemptStateUI.KILLED.toString()), String.valueOf(job.getKilledReduceAttempts()))._().td().a(url("attempts", jid, "r", TaskAttemptStateUI.SUCCESSFUL.toString()), String.valueOf(job.getSuccessfulReduceAttempts()))._()._()._()._();
}

Example 50 with JobId

use of org.apache.hadoop.mapreduce.v2.api.records.JobId in project hadoop by apache.

the class HsWebServices method getJobAttempts.

@GET
@Path("/mapreduce/jobs/{jobid}/jobattempts")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public AMAttemptsInfo getJobAttempts(@PathParam("jobid") String jid) {
    init();
    Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
    AMAttemptsInfo amAttempts = new AMAttemptsInfo();
    for (AMInfo amInfo : job.getAMInfos()) {
        AMAttemptInfo attempt = new AMAttemptInfo(amInfo, MRApps.toString(job.getID()), job.getUserName(), uriInfo.getBaseUri().toString(), webapp.name());
        amAttempts.add(attempt);
    }
    return amAttempts;
}