Examples with Counter org.apache.hadoop.mapreduce.Counter used on opensource projects

Example 36 with Counter

use of org.apache.hadoop.mapreduce.Counter in project pinot by linkedin.

the class AggregationPhaseJob method run.

public Job run() throws Exception {
    Job job = Job.getInstance(getConf());
    job.setJobName(name);
    job.setJarByClass(AggregationPhaseJob.class);
    FileSystem fs = FileSystem.get(getConf());
    Configuration configuration = job.getConfiguration();
    // Properties
    LOGGER.info("Properties {}", props);
    // Input Path
    String inputPathDir = getAndSetConfiguration(configuration, AGG_PHASE_INPUT_PATH);
    LOGGER.info("Input path dir: " + inputPathDir);
    for (String inputPath : inputPathDir.split(ThirdEyeConstants.FIELD_SEPARATOR)) {
        LOGGER.info("Adding input:" + inputPath);
        Path input = new Path(inputPath);
        FileInputFormat.addInputPath(job, input);
    }
    // Output path
    Path outputPath = new Path(getAndSetConfiguration(configuration, AGG_PHASE_OUTPUT_PATH));
    LOGGER.info("Output path dir: " + outputPath.toString());
    if (fs.exists(outputPath)) {
        fs.delete(outputPath, true);
    }
    FileOutputFormat.setOutputPath(job, outputPath);
    // Schema
    Schema avroSchema = ThirdeyeAvroUtils.getSchema(inputPathDir);
    LOGGER.info("Schema : {}", avroSchema.toString(true));
    job.getConfiguration().set(AGG_PHASE_AVRO_SCHEMA.toString(), avroSchema.toString());
    // ThirdEyeConfig
    String metricTypesProperty = ThirdeyeAvroUtils.getMetricTypesProperty(props.getProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_NAMES.toString()), props.getProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString()), avroSchema);
    props.setProperty(ThirdEyeConfigProperties.THIRDEYE_METRIC_TYPES.toString(), metricTypesProperty);
    ThirdEyeConfig thirdeyeConfig = ThirdEyeConfig.fromProperties(props);
    LOGGER.info("Thirdeye Config {}", thirdeyeConfig.encode());
    job.getConfiguration().set(AGG_PHASE_THIRDEYE_CONFIG.toString(), OBJECT_MAPPER.writeValueAsString(thirdeyeConfig));
    // Map config
    job.setMapperClass(AggregationMapper.class);
    job.setInputFormatClass(AvroKeyInputFormat.class);
    job.setMapOutputKeyClass(BytesWritable.class);
    job.setMapOutputValueClass(BytesWritable.class);
    // Reduce config
    job.setReducerClass(AggregationReducer.class);
    job.setOutputKeyClass(AvroKey.class);
    job.setOutputValueClass(NullWritable.class);
    AvroJob.setOutputKeySchema(job, avroSchema);
    job.setOutputFormatClass(AvroKeyOutputFormat.class);
    String numReducers = props.getProperty(ThirdEyeJobProperties.THIRDEYE_NUM_REDUCERS.getName());
    LOGGER.info("Num Reducers : {}", numReducers);
    if (StringUtils.isNotBlank(numReducers)) {
        job.setNumReduceTasks(Integer.valueOf(numReducers));
        LOGGER.info("Setting num reducers {}", job.getNumReduceTasks());
    }
    job.waitForCompletion(true);
    Counter counter = job.getCounters().findCounter(AggregationCounter.NUMBER_OF_RECORDS);
    LOGGER.info(counter.getDisplayName() + " : " + counter.getValue());
    if (counter.getValue() == 0) {
        throw new IllegalStateException("No input records in " + inputPathDir);
    }
    counter = job.getCounters().findCounter(AggregationCounter.NUMBER_OF_RECORDS_FLATTENED);
    LOGGER.info(counter.getDisplayName() + " : " + counter.getValue());
    for (String metric : thirdeyeConfig.getMetricNames()) {
        counter = job.getCounters().findCounter(thirdeyeConfig.getCollection(), metric);
        LOGGER.info(counter.getDisplayName() + " : " + counter.getValue());
    }
    return job;
}

Example 37 with Counter

use of org.apache.hadoop.mapreduce.Counter in project hbase by apache.

the class TableRecordReaderImpl method updateCounters.

protected static void updateCounters(ScanMetrics scanMetrics, long numScannerRestarts, Method getCounter, TaskAttemptContext context, long numStale) {
    // we can get access to counters only if hbase uses new mapreduce APIs
    if (getCounter == null) {
        return;
    }
    try {
        for (Map.Entry<String, Long> entry : scanMetrics.getMetricsMap().entrySet()) {
            Counter ct = (Counter) getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME, entry.getKey());
            ct.increment(entry.getValue());
        }
        ((Counter) getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME, "NUM_SCANNER_RESTARTS")).increment(numScannerRestarts);
        ((Counter) getCounter.invoke(context, HBASE_COUNTER_GROUP_NAME, "NUM_SCAN_RESULTS_STALE")).increment(numStale);
    } catch (Exception e) {
        LOG.debug("can't update counter." + StringUtils.stringifyException(e));
    }
}

Example 38 with Counter

use of org.apache.hadoop.mapreduce.Counter in project hadoop by apache.

the class TaskAttemptImpl method updateProgressSplits.

private void updateProgressSplits() {
    double newProgress = reportedStatus.progress;
    newProgress = Math.max(Math.min(newProgress, 1.0D), 0.0D);
    Counters counters = reportedStatus.counters;
    if (counters == null)
        return;
    WrappedProgressSplitsBlock splitsBlock = getProgressSplitBlock();
    if (splitsBlock != null) {
        long now = clock.getTime();
        // TODO Ensure not 0
        long start = getLaunchTime();
        if (start != 0 && now - start <= Integer.MAX_VALUE) {
            splitsBlock.getProgressWallclockTime().extend(newProgress, (int) (now - start));
        }
        Counter cpuCounter = counters.findCounter(TaskCounter.CPU_MILLISECONDS);
        if (cpuCounter != null && cpuCounter.getValue() <= Integer.MAX_VALUE) {
            splitsBlock.getProgressCPUTime().extend(newProgress, // long to int? TODO: FIX. Same below
            (int) cpuCounter.getValue());
        }
        Counter virtualBytes = counters.findCounter(TaskCounter.VIRTUAL_MEMORY_BYTES);
        if (virtualBytes != null) {
            splitsBlock.getProgressVirtualMemoryKbytes().extend(newProgress, (int) (virtualBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
        }
        Counter physicalBytes = counters.findCounter(TaskCounter.PHYSICAL_MEMORY_BYTES);
        if (physicalBytes != null) {
            splitsBlock.getProgressPhysicalMemoryKbytes().extend(newProgress, (int) (physicalBytes.getValue() / (MEMORY_SPLITS_RESOLUTION)));
        }
    }
}

Example 39 with Counter

use of org.apache.hadoop.mapreduce.Counter in project hadoop by apache.

the class TestTaskImpl method testCountersWithSpeculation.

@Test
public void testCountersWithSpeculation() {
    mockTask = new MockTaskImpl(jobId, partition, dispatcher.getEventHandler(), remoteJobConfFile, conf, taskAttemptListener, jobToken, credentials, clock, startCount, metrics, appContext, TaskType.MAP) {

        @Override
        protected int getMaxAttempts() {
            return 1;
        }
    };
    TaskId taskId = getNewTaskID();
    scheduleTaskAttempt(taskId);
    launchTaskAttempt(getLastAttempt().getAttemptId());
    updateLastAttemptState(TaskAttemptState.RUNNING);
    MockTaskAttemptImpl baseAttempt = getLastAttempt();
    // add a speculative attempt
    mockTask.handle(new TaskTAttemptEvent(getLastAttempt().getAttemptId(), TaskEventType.T_ADD_SPEC_ATTEMPT));
    launchTaskAttempt(getLastAttempt().getAttemptId());
    updateLastAttemptState(TaskAttemptState.RUNNING);
    MockTaskAttemptImpl specAttempt = getLastAttempt();
    assertEquals(2, taskAttempts.size());
    Counters specAttemptCounters = new Counters();
    Counter cpuCounter = specAttemptCounters.findCounter(TaskCounter.CPU_MILLISECONDS);
    cpuCounter.setValue(1000);
    specAttempt.setCounters(specAttemptCounters);
    // have the spec attempt succeed but second attempt at 1.0 progress as well
    commitTaskAttempt(specAttempt.getAttemptId());
    specAttempt.setProgress(1.0f);
    specAttempt.setState(TaskAttemptState.SUCCEEDED);
    mockTask.handle(new TaskTAttemptEvent(specAttempt.getAttemptId(), TaskEventType.T_ATTEMPT_SUCCEEDED));
    assertEquals(TaskState.SUCCEEDED, mockTask.getState());
    baseAttempt.setProgress(1.0f);
    Counters taskCounters = mockTask.getCounters();
    assertEquals("wrong counters for task", specAttemptCounters, taskCounters);
}

Example 40 with Counter

use of org.apache.hadoop.mapreduce.Counter in project hadoop by apache.

the class HumanReadableHistoryViewerPrinter method printJobCounters.

private void printJobCounters(StringBuilder buff, Counters totalCounters, Counters mapCounters, Counters reduceCounters) {
    // Killed jobs might not have counters
    if (totalCounters != null) {
        buff.append("\nCounters: \n\n");
        buff.append(String.format("|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s|", "Group Name", "Counter name", "Map Value", "Reduce Value", "Total Value"));
        buff.append("\n------------------------------------------" + "---------------------------------------------");
        for (String groupName : totalCounters.getGroupNames()) {
            CounterGroup totalGroup = totalCounters.getGroup(groupName);
            CounterGroup mapGroup = mapCounters.getGroup(groupName);
            CounterGroup reduceGroup = reduceCounters.getGroup(groupName);
            Format decimal = new DecimalFormat();
            Iterator<Counter> ctrItr = totalGroup.iterator();
            while (ctrItr.hasNext()) {
                org.apache.hadoop.mapreduce.Counter counter = ctrItr.next();
                String name = counter.getName();
                String mapValue = decimal.format(mapGroup.findCounter(name).getValue());
                String reduceValue = decimal.format(reduceGroup.findCounter(name).getValue());
                String totalValue = decimal.format(counter.getValue());
                buff.append(String.format("%n|%1$-30s|%2$-30s|%3$-10s|%4$-10s|%5$-10s", totalGroup.getDisplayName(), counter.getDisplayName(), mapValue, reduceValue, totalValue));
            }
        }
    }
}