Examples with INativeHandler org.apache.hadoop.mapred.nativetask.INativeHandler used on opensource projects

Example 1 with INativeHandler

use of org.apache.hadoop.mapred.nativetask.INativeHandler in project hadoop by apache.

the class CombinerHandler method create.

public static <K, V> ICombineHandler create(TaskContext context) throws IOException, ClassNotFoundException {
    final JobConf conf = new JobConf(context.getConf());
    conf.set(Constants.SERIALIZATION_FRAMEWORK, String.valueOf(SerializationFramework.WRITABLE_SERIALIZATION.getType()));
    String combinerClazz = conf.get(Constants.MAPRED_COMBINER_CLASS);
    if (null == combinerClazz) {
        combinerClazz = conf.get(MRJobConfig.COMBINE_CLASS_ATTR);
    }
    if (null == combinerClazz) {
        return null;
    } else {
        LOG.info("NativeTask Combiner is enabled, class = " + combinerClazz);
    }
    final Counter combineInputCounter = context.getTaskReporter().getCounter(TaskCounter.COMBINE_INPUT_RECORDS);
    final CombinerRunner<K, V> combinerRunner = CombinerRunner.create(conf, context.getTaskAttemptId(), combineInputCounter, context.getTaskReporter(), null);
    final INativeHandler nativeHandler = NativeBatchProcessor.create(NAME, conf, DataChannel.INOUT);
    @SuppressWarnings("unchecked") final BufferPusher<K, V> pusher = new BufferPusher<K, V>((Class<K>) context.getInputKeyClass(), (Class<V>) context.getInputValueClass(), nativeHandler);
    final BufferPuller puller = new BufferPuller(nativeHandler);
    return new CombinerHandler<K, V>(nativeHandler, combinerRunner, puller, pusher);
}

Example 2 with INativeHandler

use of org.apache.hadoop.mapred.nativetask.INativeHandler in project hadoop by apache.

the class NativeCollectorOnlyHandler method create.

public static <K, V> NativeCollectorOnlyHandler<K, V> create(TaskContext context) throws IOException {
    ICombineHandler combinerHandler = null;
    try {
        final TaskContext combineContext = context.copyOf();
        combineContext.setInputKeyClass(context.getOutputKeyClass());
        combineContext.setInputValueClass(context.getOutputValueClass());
        combinerHandler = CombinerHandler.create(combineContext);
    } catch (final ClassNotFoundException e) {
        throw new IOException(e);
    }
    if (null != combinerHandler) {
        LOG.info("[NativeCollectorOnlyHandler] combiner is not null");
    }
    final INativeHandler nativeHandler = NativeBatchProcessor.create(NAME, context.getConf(), DataChannel.OUT);
    final BufferPusher<K, V> kvPusher = new BufferPusher<K, V>((Class<K>) context.getOutputKeyClass(), (Class<V>) context.getOutputValueClass(), nativeHandler);
    return new NativeCollectorOnlyHandler<K, V>(context, nativeHandler, kvPusher, combinerHandler);
}