Examples with CounterGroup org.apache.hadoop.mapreduce.CounterGroup used on opensource projects

Example 11 with CounterGroup

use of org.apache.hadoop.mapreduce.CounterGroup in project hadoop by apache.

the class JSONHistoryViewerPrinter method printTaskCounters.

private void printTaskCounters(JSONObject jTask, Counters taskCounters) throws JSONException {
    // Killed tasks might not have counters
    if (taskCounters != null) {
        JSONObject jGroups = new JSONObject();
        for (String groupName : taskCounters.getGroupNames()) {
            CounterGroup group = taskCounters.getGroup(groupName);
            Iterator<Counter> ctrItr = group.iterator();
            JSONArray jGroup = new JSONArray();
            while (ctrItr.hasNext()) {
                JSONObject jCounter = new JSONObject();
                org.apache.hadoop.mapreduce.Counter counter = ctrItr.next();
                jCounter.put("counterName", counter.getName());
                jCounter.put("value", counter.getValue());
                jGroup.put(jCounter);
            }
            jGroups.put(fixGroupNameForShuffleErrors(group.getName()), jGroup);
        }
        jTask.put("counters", jGroups);
    }
}

Example 12 with CounterGroup

use of org.apache.hadoop.mapreduce.CounterGroup in project hadoop by apache.

the class JobHistoryEventUtils method countersToJSON.

public static JsonNode countersToJSON(Counters counters) {
    ObjectMapper mapper = new ObjectMapper();
    ArrayNode nodes = mapper.createArrayNode();
    if (counters != null) {
        for (CounterGroup counterGroup : counters) {
            ObjectNode groupNode = nodes.addObject();
            groupNode.put("NAME", counterGroup.getName());
            groupNode.put("DISPLAY_NAME", counterGroup.getDisplayName());
            ArrayNode countersNode = groupNode.putArray("COUNTERS");
            for (Counter counter : counterGroup) {
                ObjectNode counterNode = countersNode.addObject();
                counterNode.put("NAME", counter.getName());
                counterNode.put("DISPLAY_NAME", counter.getDisplayName());
                counterNode.put("VALUE", counter.getValue());
            }
        }
    }
    return nodes;
}

Example 13 with CounterGroup

use of org.apache.hadoop.mapreduce.CounterGroup in project hadoop by apache.

the class TestMRMultipleOutputs method _testMOWithJavaSerialization.

protected void _testMOWithJavaSerialization(boolean withCounters) throws Exception {
    String input = "a\nb\nc\nd\ne\nc\nd\ne";
    Configuration conf = createJobConf();
    conf.set("io.serializations", "org.apache.hadoop.io.serializer.JavaSerialization," + "org.apache.hadoop.io.serializer.WritableSerialization");
    Job job = MapReduceTestUtil.createJob(conf, IN_DIR, OUT_DIR, 2, 1, input);
    job.setJobName("mo");
    MultipleOutputs.addNamedOutput(job, TEXT, TextOutputFormat.class, Long.class, String.class);
    MultipleOutputs.setCountersEnabled(job, withCounters);
    job.setSortComparatorClass(JavaSerializationComparator.class);
    job.setMapOutputKeyClass(Long.class);
    job.setMapOutputValueClass(String.class);
    job.setOutputKeyClass(Long.class);
    job.setOutputValueClass(String.class);
    job.setMapperClass(MOJavaSerDeMap.class);
    job.setReducerClass(MOJavaSerDeReduce.class);
    job.waitForCompletion(true);
    // assert number of named output part files
    int namedOutputCount = 0;
    int valueBasedOutputCount = 0;
    FileSystem fs = OUT_DIR.getFileSystem(conf);
    FileStatus[] statuses = fs.listStatus(OUT_DIR);
    for (FileStatus status : statuses) {
        String fileName = status.getPath().getName();
        if (fileName.equals("text-m-00000") || fileName.equals("text-m-00001") || fileName.equals("text-r-00000")) {
            namedOutputCount++;
        } else if (fileName.equals("a-r-00000") || fileName.equals("b-r-00000") || fileName.equals("c-r-00000") || fileName.equals("d-r-00000") || fileName.equals("e-r-00000")) {
            valueBasedOutputCount++;
        }
    }
    assertEquals(3, namedOutputCount);
    assertEquals(5, valueBasedOutputCount);
    // assert TextOutputFormat files correctness
    BufferedReader reader = new BufferedReader(new InputStreamReader(fs.open(new Path(FileOutputFormat.getOutputPath(job), "text-r-00000"))));
    int count = 0;
    String line = reader.readLine();
    while (line != null) {
        assertTrue(line.endsWith(TEXT));
        line = reader.readLine();
        count++;
    }
    reader.close();
    assertFalse(count == 0);
    if (withCounters) {
        CounterGroup counters = job.getCounters().getGroup(MultipleOutputs.class.getName());
        assertEquals(6, counters.size());
        assertEquals(4, counters.findCounter(TEXT).getValue());
        assertEquals(2, counters.findCounter("a").getValue());
        assertEquals(2, counters.findCounter("b").getValue());
        assertEquals(4, counters.findCounter("c").getValue());
        assertEquals(4, counters.findCounter("d").getValue());
        assertEquals(4, counters.findCounter("e").getValue());
    }
}