Examples with Counters org.apache.hadoop.mapreduce.v2.api.records.Counters used on opensource projects

Example 31 with Counters

use of org.apache.hadoop.mapreduce.v2.api.records.Counters in project hadoop by apache.

the class TestMRJobsWithHistoryService method testJobHistoryData.

@Test(timeout = 90000)
public void testJobHistoryData() throws IOException, InterruptedException, AvroRemoteException, ClassNotFoundException {
    if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
        LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR + " not found. Not running test.");
        return;
    }
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(mrCluster.getConfig());
    // Job with 3 maps and 2 reduces
    Job job = sleepJob.createJob(3, 2, 1000, 1, 500, 1);
    job.setJarByClass(SleepJob.class);
    // The AppMaster jar itself.
    job.addFileToClassPath(APP_JAR);
    job.waitForCompletion(true);
    Counters counterMR = job.getCounters();
    JobId jobId = TypeConverter.toYarn(job.getJobID());
    ApplicationId appID = jobId.getAppId();
    int pollElapsed = 0;
    while (true) {
        Thread.sleep(1000);
        pollElapsed += 1000;
        if (TERMINAL_RM_APP_STATES.contains(mrCluster.getResourceManager().getRMContext().getRMApps().get(appID).getState())) {
            break;
        }
        if (pollElapsed >= 60000) {
            LOG.warn("application did not reach terminal state within 60 seconds");
            break;
        }
    }
    Assert.assertEquals(RMAppState.FINISHED, mrCluster.getResourceManager().getRMContext().getRMApps().get(appID).getState());
    Counters counterHS = job.getCounters();
    //TODO the Assert below worked. need to check
    //Should we compare each field or convert to V2 counter and compare
    LOG.info("CounterHS " + counterHS);
    LOG.info("CounterMR " + counterMR);
    Assert.assertEquals(counterHS, counterMR);
    HSClientProtocol historyClient = instantiateHistoryProxy();
    GetJobReportRequest gjReq = Records.newRecord(GetJobReportRequest.class);
    gjReq.setJobId(jobId);
    JobReport jobReport = historyClient.getJobReport(gjReq).getJobReport();
    verifyJobReport(jobReport, jobId);
}

Example 32 with Counters

use of org.apache.hadoop.mapreduce.v2.api.records.Counters in project hadoop by apache.

the class HsWebServices method getJobTaskAttemptIdCounters.

@GET
@Path("/mapreduce/jobs/{jobid}/tasks/{taskid}/attempts/{attemptid}/counters")
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8, MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
public JobTaskAttemptCounterInfo getJobTaskAttemptIdCounters(@Context HttpServletRequest hsr, @PathParam("jobid") String jid, @PathParam("taskid") String tid, @PathParam("attemptid") String attId) {
    init();
    Job job = AMWebServices.getJobFromJobIdString(jid, ctx);
    checkAccess(job, hsr);
    Task task = AMWebServices.getTaskFromTaskIdString(tid, job);
    TaskAttempt ta = AMWebServices.getTaskAttemptFromTaskAttemptString(attId, task);
    return new JobTaskAttemptCounterInfo(ta);
}

Example 33 with Counters

use of org.apache.hadoop.mapreduce.v2.api.records.Counters in project hadoop by apache.

the class TaskAttemptListenerImpl method statusUpdate.

@Override
public AMFeedback statusUpdate(TaskAttemptID taskAttemptID, TaskStatus taskStatus) throws IOException, InterruptedException {
    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId yarnAttemptID = TypeConverter.toYarn(taskAttemptID);
    AMFeedback feedback = new AMFeedback();
    feedback.setTaskFound(true);
    // Propagating preemption to the task if TASK_PREEMPTION is enabled
    if (getConfig().getBoolean(MRJobConfig.TASK_PREEMPTION, false) && preemptionPolicy.isPreempted(yarnAttemptID)) {
        feedback.setPreemption(true);
        LOG.info("Setting preemption bit for task: " + yarnAttemptID + " of type " + yarnAttemptID.getTaskId().getTaskType());
    }
    if (taskStatus == null) {
        //We are using statusUpdate only as a simple ping
        if (LOG.isDebugEnabled()) {
            LOG.debug("Ping from " + taskAttemptID.toString());
        }
        return feedback;
    }
    // if we are here there is an actual status update to be processed
    taskHeartbeatHandler.progressing(yarnAttemptID);
    TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatus();
    taskAttemptStatus.id = yarnAttemptID;
    // Task sends the updated progress to the TT.
    taskAttemptStatus.progress = taskStatus.getProgress();
    LOG.info("Progress of TaskAttempt " + taskAttemptID + " is : " + taskStatus.getProgress());
    // Task sends the updated state-string to the TT.
    taskAttemptStatus.stateString = taskStatus.getStateString();
    // Task sends the updated phase to the TT.
    taskAttemptStatus.phase = TypeConverter.toYarn(taskStatus.getPhase());
    // Counters are updated by the task. Convert counters into new format as
    // that is the primary storage format inside the AM to avoid multiple
    // conversions and unnecessary heap usage.
    taskAttemptStatus.counters = new org.apache.hadoop.mapreduce.Counters(taskStatus.getCounters());
    // Map Finish time set by the task (map only)
    if (taskStatus.getIsMap() && taskStatus.getMapFinishTime() != 0) {
        taskAttemptStatus.mapFinishTime = taskStatus.getMapFinishTime();
    }
    // Shuffle Finish time set by the task (reduce only).
    if (!taskStatus.getIsMap() && taskStatus.getShuffleFinishTime() != 0) {
        taskAttemptStatus.shuffleFinishTime = taskStatus.getShuffleFinishTime();
    }
    // Sort finish time set by the task (reduce only).
    if (!taskStatus.getIsMap() && taskStatus.getSortFinishTime() != 0) {
        taskAttemptStatus.sortFinishTime = taskStatus.getSortFinishTime();
    }
    //set the fetch failures
    if (taskStatus.getFetchFailedMaps() != null && taskStatus.getFetchFailedMaps().size() > 0) {
        taskAttemptStatus.fetchFailedMaps = new ArrayList<org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId>();
        for (TaskAttemptID failedMapId : taskStatus.getFetchFailedMaps()) {
            taskAttemptStatus.fetchFailedMaps.add(TypeConverter.toYarn(failedMapId));
        }
    }
    // Task sends the information about the nextRecordRange to the TT
    //    TODO: The following are not needed here, but needed to be set somewhere inside AppMaster.
    //    taskStatus.getRunState(); // Set by the TT/JT. Transform into a state TODO
    //    taskStatus.getStartTime(); // Used to be set by the TaskTracker. This should be set by getTask().
    //    taskStatus.getFinishTime(); // Used to be set by TT/JT. Should be set when task finishes
    //    // This was used by TT to do counter updates only once every minute. So this
    //    // isn't ever changed by the Task itself.
    //    taskStatus.getIncludeCounters();
    context.getEventHandler().handle(new TaskAttemptStatusUpdateEvent(taskAttemptStatus.id, taskAttemptStatus));
    return feedback;
}

Example 34 with Counters

use of org.apache.hadoop.mapreduce.v2.api.records.Counters in project hadoop by apache.

the class TaskImpl method getReport.

@Override
public TaskReport getReport() {
    TaskReport report = recordFactory.newRecordInstance(TaskReport.class);
    readLock.lock();
    try {
        TaskAttempt bestAttempt = selectBestAttempt();
        report.setTaskId(taskId);
        report.setStartTime(getLaunchTime());
        report.setFinishTime(getFinishTime());
        report.setTaskState(getState());
        report.setProgress(bestAttempt == null ? 0f : bestAttempt.getProgress());
        report.setStatus(bestAttempt == null ? "" : bestAttempt.getReport().getStateString());
        for (TaskAttempt attempt : attempts.values()) {
            if (TaskAttemptState.RUNNING.equals(attempt.getState())) {
                report.addRunningAttempt(attempt.getID());
            }
        }
        report.setSuccessfulAttempt(successfulAttempt);
        for (TaskAttempt att : attempts.values()) {
            String prefix = "AttemptID:" + att.getID() + " Info:";
            for (CharSequence cs : att.getDiagnostics()) {
                report.addDiagnostics(prefix + cs);
            }
        }
        // Add a copy of counters as the last step so that their lifetime on heap
        // is as small as possible.
        report.setCounters(TypeConverter.toYarn(bestAttempt == null ? TaskAttemptImpl.EMPTY_COUNTERS : bestAttempt.getCounters()));
        return report;
    } finally {
        readLock.unlock();
    }
}

Example 35 with Counters

use of org.apache.hadoop.mapreduce.v2.api.records.Counters in project hadoop by apache.

the class JobImpl method getAllCounters.

@Override
public Counters getAllCounters() {
    readLock.lock();
    try {
        JobStateInternal state = getInternalState();
        if (state == JobStateInternal.ERROR || state == JobStateInternal.FAILED || state == JobStateInternal.KILLED || state == JobStateInternal.SUCCEEDED) {
            this.mayBeConstructFinalFullCounters();
            return fullCounters;
        }
        Counters counters = new Counters();
        counters.incrAllCounters(jobCounters);
        return incrTaskCounters(counters, tasks.values());
    } finally {
        readLock.unlock();
    }
}