Examples with Counters org.apache.hadoop.mapreduce.Counters used on opensource projects

Example 6 with Counters

use of org.apache.hadoop.mapreduce.Counters in project hadoop by apache.

the class ResultVerifier method verifyCounters.

public static void verifyCounters(Job normalJob, Job nativeJob, boolean hasCombiner) throws IOException {
    Counters normalCounters = normalJob.getCounters();
    Counters nativeCounters = nativeJob.getCounters();
    assertEquals("Counter MAP_OUTPUT_RECORDS should be equal", normalCounters.findCounter(TaskCounter.MAP_OUTPUT_RECORDS).getValue(), nativeCounters.findCounter(TaskCounter.MAP_OUTPUT_RECORDS).getValue());
    assertEquals("Counter REDUCE_INPUT_GROUPS should be equal", normalCounters.findCounter(TaskCounter.REDUCE_INPUT_GROUPS).getValue(), nativeCounters.findCounter(TaskCounter.REDUCE_INPUT_GROUPS).getValue());
    if (!hasCombiner) {
        assertEquals("Counter REDUCE_INPUT_RECORDS should be equal", normalCounters.findCounter(TaskCounter.REDUCE_INPUT_RECORDS).getValue(), nativeCounters.findCounter(TaskCounter.REDUCE_INPUT_RECORDS).getValue());
    }
}

Example 7 with Counters

use of org.apache.hadoop.mapreduce.Counters in project hadoop by apache.

the class TestHistoryViewerPrinter method createCounters.

private static Counters createCounters() {
    Counters counters = new Counters();
    counters.findCounter("group1", "counter1").setValue(5);
    counters.findCounter("group1", "counter2").setValue(10);
    counters.findCounter("group2", "counter1").setValue(15);
    return counters;
}

Example 8 with Counters

use of org.apache.hadoop.mapreduce.Counters in project hadoop by apache.

the class TestFetchFailure method updateStatus.

private void updateStatus(MRApp app, TaskAttempt attempt, Phase phase) {
    TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus();
    status.counters = new Counters();
    status.fetchFailedMaps = new ArrayList<TaskAttemptId>();
    status.id = attempt.getID();
    status.mapFinishTime = 0;
    status.phase = phase;
    status.progress = 0.5f;
    status.shuffleFinishTime = 0;
    status.sortFinishTime = 0;
    status.stateString = "OK";
    status.taskState = attempt.getState();
    TaskAttemptStatusUpdateEvent event = new TaskAttemptStatusUpdateEvent(attempt.getID(), status);
    app.getContext().getEventHandler().handle(event);
}

Example 9 with Counters

use of org.apache.hadoop.mapreduce.Counters in project hadoop by apache.

the class JHEventHandlerForSigtermTest method mockJob.

private Job mockJob() {
    Job mockJob = mock(Job.class);
    when(mockJob.getAllCounters()).thenReturn(new Counters());
    when(mockJob.getTotalMaps()).thenReturn(10);
    when(mockJob.getTotalReduces()).thenReturn(10);
    when(mockJob.getName()).thenReturn("mockjob");
    return mockJob;
}

Example 10 with Counters

use of org.apache.hadoop.mapreduce.Counters in project hadoop by apache.

the class JHEventHandlerForSigtermTest method testCountersToJSONEmpty.

@Test(timeout = 50000)
public void testCountersToJSONEmpty() throws Exception {
    JobHistoryEventHandler jheh = new JobHistoryEventHandler(null, 0);
    Counters counters = null;
    JsonNode jsonNode = JobHistoryEventUtils.countersToJSON(counters);
    String jsonStr = new ObjectMapper().writeValueAsString(jsonNode);
    String expected = "[]";
    Assert.assertEquals(expected, jsonStr);
    counters = new Counters();
    jsonNode = JobHistoryEventUtils.countersToJSON(counters);
    jsonStr = new ObjectMapper().writeValueAsString(jsonNode);
    expected = "[]";
    Assert.assertEquals(expected, jsonStr);
    counters.addGroup("DOCTORS", "Incarnations of the Doctor");
    jsonNode = JobHistoryEventUtils.countersToJSON(counters);
    jsonStr = new ObjectMapper().writeValueAsString(jsonNode);
    expected = "[{\"NAME\":\"DOCTORS\",\"DISPLAY_NAME\":\"Incarnations of the " + "Doctor\",\"COUNTERS\":[]}]";
    Assert.assertEquals(expected, jsonStr);
}