Examples with Counter org.apache.hadoop.mapreduce.v2.api.records.Counter used on opensource projects

Example 6 with Counter

use of org.apache.hadoop.mapreduce.v2.api.records.Counter in project hadoop by apache.

the class TestClientRedirect method getMyCounters.

static Counters getMyCounters() {
    Counter counter = recordFactory.newRecordInstance(Counter.class);
    counter.setName("Mycounter");
    counter.setDisplayName("My counter display name");
    counter.setValue(12345);
    CounterGroup group = recordFactory.newRecordInstance(CounterGroup.class);
    group.setName("MyGroup");
    group.setDisplayName("My groupd display name");
    group.setCounter("myCounter", counter);
    Counters counters = recordFactory.newRecordInstance(Counters.class);
    counters.setCounterGroup("myGroupd", group);
    return counters;
}

Example 7 with Counter

use of org.apache.hadoop.mapreduce.v2.api.records.Counter in project hadoop by apache.

the class TestMRJobsWithHistoryService method testJobHistoryData.

@Test(timeout = 90000)
public void testJobHistoryData() throws IOException, InterruptedException, AvroRemoteException, ClassNotFoundException {
    if (!(new File(MiniMRYarnCluster.APPJAR)).exists()) {
        LOG.info("MRAppJar " + MiniMRYarnCluster.APPJAR + " not found. Not running test.");
        return;
    }
    SleepJob sleepJob = new SleepJob();
    sleepJob.setConf(mrCluster.getConfig());
    // Job with 3 maps and 2 reduces
    Job job = sleepJob.createJob(3, 2, 1000, 1, 500, 1);
    job.setJarByClass(SleepJob.class);
    // The AppMaster jar itself.
    job.addFileToClassPath(APP_JAR);
    job.waitForCompletion(true);
    Counters counterMR = job.getCounters();
    JobId jobId = TypeConverter.toYarn(job.getJobID());
    ApplicationId appID = jobId.getAppId();
    int pollElapsed = 0;
    while (true) {
        Thread.sleep(1000);
        pollElapsed += 1000;
        if (TERMINAL_RM_APP_STATES.contains(mrCluster.getResourceManager().getRMContext().getRMApps().get(appID).getState())) {
            break;
        }
        if (pollElapsed >= 60000) {
            LOG.warn("application did not reach terminal state within 60 seconds");
            break;
        }
    }
    Assert.assertEquals(RMAppState.FINISHED, mrCluster.getResourceManager().getRMContext().getRMApps().get(appID).getState());
    Counters counterHS = job.getCounters();
    //TODO the Assert below worked. need to check
    //Should we compare each field or convert to V2 counter and compare
    LOG.info("CounterHS " + counterHS);
    LOG.info("CounterMR " + counterMR);
    Assert.assertEquals(counterHS, counterMR);
    HSClientProtocol historyClient = instantiateHistoryProxy();
    GetJobReportRequest gjReq = Records.newRecord(GetJobReportRequest.class);
    gjReq.setJobId(jobId);
    JobReport jobReport = historyClient.getJobReport(gjReq).getJobReport();
    verifyJobReport(jobReport, jobId);
}

Example 8 with Counter

use of org.apache.hadoop.mapreduce.v2.api.records.Counter in project hadoop by apache.

the class TaskAttemptListenerImpl method statusUpdate.

@Override
public AMFeedback statusUpdate(TaskAttemptID taskAttemptID, TaskStatus taskStatus) throws IOException, InterruptedException {
    org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId yarnAttemptID = TypeConverter.toYarn(taskAttemptID);
    AMFeedback feedback = new AMFeedback();
    feedback.setTaskFound(true);
    // Propagating preemption to the task if TASK_PREEMPTION is enabled
    if (getConfig().getBoolean(MRJobConfig.TASK_PREEMPTION, false) && preemptionPolicy.isPreempted(yarnAttemptID)) {
        feedback.setPreemption(true);
        LOG.info("Setting preemption bit for task: " + yarnAttemptID + " of type " + yarnAttemptID.getTaskId().getTaskType());
    }
    if (taskStatus == null) {
        //We are using statusUpdate only as a simple ping
        if (LOG.isDebugEnabled()) {
            LOG.debug("Ping from " + taskAttemptID.toString());
        }
        return feedback;
    }
    // if we are here there is an actual status update to be processed
    taskHeartbeatHandler.progressing(yarnAttemptID);
    TaskAttemptStatus taskAttemptStatus = new TaskAttemptStatus();
    taskAttemptStatus.id = yarnAttemptID;
    // Task sends the updated progress to the TT.
    taskAttemptStatus.progress = taskStatus.getProgress();
    LOG.info("Progress of TaskAttempt " + taskAttemptID + " is : " + taskStatus.getProgress());
    // Task sends the updated state-string to the TT.
    taskAttemptStatus.stateString = taskStatus.getStateString();
    // Task sends the updated phase to the TT.
    taskAttemptStatus.phase = TypeConverter.toYarn(taskStatus.getPhase());
    // Counters are updated by the task. Convert counters into new format as
    // that is the primary storage format inside the AM to avoid multiple
    // conversions and unnecessary heap usage.
    taskAttemptStatus.counters = new org.apache.hadoop.mapreduce.Counters(taskStatus.getCounters());
    // Map Finish time set by the task (map only)
    if (taskStatus.getIsMap() && taskStatus.getMapFinishTime() != 0) {
        taskAttemptStatus.mapFinishTime = taskStatus.getMapFinishTime();
    }
    // Shuffle Finish time set by the task (reduce only).
    if (!taskStatus.getIsMap() && taskStatus.getShuffleFinishTime() != 0) {
        taskAttemptStatus.shuffleFinishTime = taskStatus.getShuffleFinishTime();
    }
    // Sort finish time set by the task (reduce only).
    if (!taskStatus.getIsMap() && taskStatus.getSortFinishTime() != 0) {
        taskAttemptStatus.sortFinishTime = taskStatus.getSortFinishTime();
    }
    //set the fetch failures
    if (taskStatus.getFetchFailedMaps() != null && taskStatus.getFetchFailedMaps().size() > 0) {
        taskAttemptStatus.fetchFailedMaps = new ArrayList<org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId>();
        for (TaskAttemptID failedMapId : taskStatus.getFetchFailedMaps()) {
            taskAttemptStatus.fetchFailedMaps.add(TypeConverter.toYarn(failedMapId));
        }
    }
    // Task sends the information about the nextRecordRange to the TT
    //    TODO: The following are not needed here, but needed to be set somewhere inside AppMaster.
    //    taskStatus.getRunState(); // Set by the TT/JT. Transform into a state TODO
    //    taskStatus.getStartTime(); // Used to be set by the TaskTracker. This should be set by getTask().
    //    taskStatus.getFinishTime(); // Used to be set by TT/JT. Should be set when task finishes
    //    // This was used by TT to do counter updates only once every minute. So this
    //    // isn't ever changed by the Task itself.
    //    taskStatus.getIncludeCounters();
    context.getEventHandler().handle(new TaskAttemptStatusUpdateEvent(taskAttemptStatus.id, taskAttemptStatus));
    return feedback;
}

Example 9 with Counter

use of org.apache.hadoop.mapreduce.v2.api.records.Counter in project hadoop by apache.

the class TestTaskImpl method testCountersWithSpeculation.

@Test
public void testCountersWithSpeculation() {
    mockTask = new MockTaskImpl(jobId, partition, dispatcher.getEventHandler(), remoteJobConfFile, conf, taskAttemptListener, jobToken, credentials, clock, startCount, metrics, appContext, TaskType.MAP) {

        @Override
        protected int getMaxAttempts() {
            return 1;
        }
    };
    TaskId taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    launchTaskAttempt(getLastAttempt().getAttemptId());
    updateLastAttemptState(TaskAttemptState.RUNNING);
    MockTaskAttemptImpl baseAttempt = getLastAttempt();
    // add a speculative attempt
    mockTask.handle(new TaskTAttemptEvent(getLastAttempt().getAttemptId(), TaskEventType.T_ADD_SPEC_ATTEMPT));
    launchTaskAttempt(getLastAttempt().getAttemptId());
    updateLastAttemptState(TaskAttemptState.RUNNING);
    MockTaskAttemptImpl specAttempt = getLastAttempt();
    assertEquals(2, taskAttempts.size());
    Counters specAttemptCounters = new Counters();
    Counter cpuCounter = specAttemptCounters.findCounter(TaskCounter.CPU_MILLISECONDS);
    cpuCounter.setValue(1000);
    specAttempt.setCounters(specAttemptCounters);
    // have the spec attempt succeed but second attempt at 1.0 progress as well
    commitTaskAttempt(specAttempt.getAttemptId());
    specAttempt.setProgress(1.0f);
    specAttempt.setState(TaskAttemptState.SUCCEEDED);
    mockTask.handle(new TaskTAttemptEvent(specAttempt.getAttemptId(), TaskEventType.T_ATTEMPT_SUCCEEDED));
    assertEquals(TaskState.SUCCEEDED, mockTask.getState());
    baseAttempt.setProgress(1.0f);
    Counters taskCounters = mockTask.getCounters();
    assertEquals("wrong counters for task", specAttemptCounters, taskCounters);
}

Example 10 with Counter

use of org.apache.hadoop.mapreduce.v2.api.records.Counter in project hadoop by apache.

the class TypeConverter method toYarn.

public static Counters toYarn(org.apache.hadoop.mapreduce.Counters counters) {
    if (counters == null) {
        return null;
    }
    Counters yCntrs = recordFactory.newRecordInstance(Counters.class);
    yCntrs.addAllCounterGroups(new HashMap<String, CounterGroup>());
    for (org.apache.hadoop.mapreduce.CounterGroup grp : counters) {
        CounterGroup yGrp = recordFactory.newRecordInstance(CounterGroup.class);
        yGrp.setName(grp.getName());
        yGrp.setDisplayName(grp.getDisplayName());
        yGrp.addAllCounters(new HashMap<String, Counter>());
        for (org.apache.hadoop.mapreduce.Counter cntr : grp) {
            Counter yCntr = recordFactory.newRecordInstance(Counter.class);
            yCntr.setName(cntr.getName());
            yCntr.setDisplayName(cntr.getDisplayName());
            yCntr.setValue(cntr.getValue());
            yGrp.setCounter(yCntr.getName(), yCntr);
        }
        yCntrs.setCounterGroup(yGrp.getName(), yGrp);
    }
    return yCntrs;
}