Examples with Counter org.apache.hadoop.mapreduce.Counter used on opensource projects

Example 1 with Counter

use of org.apache.hadoop.mapreduce.Counter in project druid by druid-io.

the class IndexGeneratorJob method run.

public boolean run() {
    try {
        Job job = Job.getInstance(new Configuration(), String.format("%s-index-generator-%s", config.getDataSource(), config.getIntervals()));
        job.getConfiguration().set("io.sort.record.percent", "0.23");
        JobHelper.injectSystemProperties(job);
        config.addJobProperties(job);
        job.setMapperClass(IndexGeneratorMapper.class);
        job.setMapOutputValueClass(BytesWritable.class);
        SortableBytes.useSortableBytesAsMapOutputKey(job);
        int numReducers = Iterables.size(config.getAllBuckets().get());
        if (numReducers == 0) {
            throw new RuntimeException("No buckets?? seems there is no data to index.");
        }
        if (config.getSchema().getTuningConfig().getUseCombiner()) {
            job.setCombinerClass(IndexGeneratorCombiner.class);
            job.setCombinerKeyGroupingComparatorClass(BytesWritable.Comparator.class);
        }
        job.setNumReduceTasks(numReducers);
        job.setPartitionerClass(IndexGeneratorPartitioner.class);
        setReducerClass(job);
        job.setOutputKeyClass(BytesWritable.class);
        job.setOutputValueClass(Text.class);
        job.setOutputFormatClass(IndexGeneratorOutputFormat.class);
        FileOutputFormat.setOutputPath(job, config.makeIntermediatePath());
        config.addInputPaths(job);
        config.intoConfiguration(job);
        JobHelper.setupClasspath(JobHelper.distributedClassPath(config.getWorkingPath()), JobHelper.distributedClassPath(config.makeIntermediatePath()), job);
        job.submit();
        log.info("Job %s submitted, status available at %s", job.getJobName(), job.getTrackingURL());
        boolean success = job.waitForCompletion(true);
        Counter invalidRowCount = job.getCounters().findCounter(HadoopDruidIndexerConfig.IndexJobCounters.INVALID_ROW_COUNTER);
        jobStats.setInvalidRowCount(invalidRowCount.getValue());
        return success;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 2 with Counter

use of org.apache.hadoop.mapreduce.Counter in project hbase by apache.

the class TestRowCounter method runRowCount.

/**
   * Run the RowCounter map reduce job and verify the row count.
   *
   * @param args the command line arguments to be used for rowcounter job.
   * @param expectedCount the expected row count (result of map reduce job).
   * @throws Exception
   */
private void runRowCount(String[] args, int expectedCount) throws Exception {
    Job job = RowCounter.createSubmittableJob(TEST_UTIL.getConfiguration(), args);
    long start = System.currentTimeMillis();
    job.waitForCompletion(true);
    long duration = System.currentTimeMillis() - start;
    LOG.debug("row count duration (ms): " + duration);
    assertTrue(job.isSuccessful());
    Counter counter = job.getCounters().findCounter(RowCounter.RowCounterMapper.Counters.ROWS);
    assertEquals(expectedCount, counter.getValue());
}

Example 3 with Counter

use of org.apache.hadoop.mapreduce.Counter in project pinot by linkedin.

the class JoinPhaseJob method dumpSummary.

private void dumpSummary(Job job, List<String> sourceNames) throws IOException {
    System.out.println("Join Input Matrix.");
    CounterGroup group = job.getCounters().getGroup("DynamicCounter");
    for (String source : sourceNames) {
        System.out.print(String.format("%25s\t", source));
    }
    if (group != null) {
        Iterator<Counter> iterator = group.iterator();
        while (iterator.hasNext()) {
            Counter counter = iterator.next();
            String displayName = counter.getDisplayName();
            String[] split = displayName.replace("[", "").replace("[", "").split(",");
            for (String str : split) {
                if (str.trim().equals("1")) {
                    System.out.print(String.format("%25s\t", "1"));
                } else {
                    System.out.print(String.format("%25s\t", "-"));
                }
            }
        }
    }
}

Example 4 with Counter

use of org.apache.hadoop.mapreduce.Counter in project hadoop by apache.

the class AbstractCounterGroup method write.

/**
   * GenericGroup ::= displayName #counter counter*
   */
@Override
public synchronized void write(DataOutput out) throws IOException {
    Text.writeString(out, displayName);
    WritableUtils.writeVInt(out, counters.size());
    for (Counter counter : counters.values()) {
        counter.write(out);
    }
}

Example 5 with Counter

use of org.apache.hadoop.mapreduce.Counter in project hadoop by apache.

the class EventWriter method toAvro.

static JhCounters toAvro(Counters counters, String name) {
    JhCounters result = new JhCounters();
    result.setName(new Utf8(name));
    result.setGroups(new ArrayList<JhCounterGroup>(0));
    if (counters == null)
        return result;
    for (CounterGroup group : counters) {
        JhCounterGroup g = new JhCounterGroup();
        g.setName(new Utf8(group.getName()));
        g.setDisplayName(new Utf8(group.getDisplayName()));
        g.setCounts(new ArrayList<JhCounter>(group.size()));
        for (Counter counter : group) {
            JhCounter c = new JhCounter();
            c.setName(new Utf8(counter.getName()));
            c.setDisplayName(new Utf8(counter.getDisplayName()));
            c.setValue(counter.getValue());
            g.getCounts().add(c);
        }
        result.getGroups().add(g);
    }
    return result;
}