Examples with KeyValueWriter org.apache.tez.runtime.library.api.KeyValueWriter used on opensource projects

Example 6 with KeyValueWriter

use of org.apache.tez.runtime.library.api.KeyValueWriter in project tez by apache.

the class MapProcessor method run.

@Override
public void run(Map<String, LogicalInput> _inputs, Map<String, LogicalOutput> _outputs) throws Exception {
    this.inputs = _inputs;
    this.outputs = _outputs;
    progressHelper = new ProgressHelper(this.inputs, getContext(), this.getClass().getSimpleName());
    LOG.info("Running map: " + processorContext.getUniqueIdentifier());
    if (_inputs.size() != 1 || _outputs.size() != 1) {
        throw new IOException("Cannot handle multiple _inputs or _outputs" + ", inputCount=" + _inputs.size() + ", outputCount=" + _outputs.size());
    }
    for (LogicalInput input : _inputs.values()) {
        input.start();
    }
    for (LogicalOutput output : _outputs.values()) {
        output.start();
    }
    LogicalInput in = _inputs.values().iterator().next();
    LogicalOutput out = _outputs.values().iterator().next();
    initTask(out);
    progressHelper.scheduleProgressTaskService(0, 100);
    // Sanity check
    if (!(in instanceof MRInputLegacy)) {
        throw new IOException(new TezException("Only MRInputLegacy supported. Input: " + in.getClass()));
    }
    MRInputLegacy input = (MRInputLegacy) in;
    input.init();
    Configuration incrementalConf = input.getConfigUpdates();
    if (incrementalConf != null) {
        for (Entry<String, String> entry : incrementalConf) {
            jobConf.set(entry.getKey(), entry.getValue());
        }
    }
    KeyValueWriter kvWriter = null;
    if ((out instanceof MROutputLegacy)) {
        kvWriter = ((MROutputLegacy) out).getWriter();
    } else if ((out instanceof OrderedPartitionedKVOutput)) {
        kvWriter = ((OrderedPartitionedKVOutput) out).getWriter();
    } else {
        throw new IOException("Illegal output to map, outputClass=" + out.getClass());
    }
    if (useNewApi) {
        runNewMapper(jobConf, mrReporter, input, kvWriter);
    } else {
        runOldMapper(jobConf, mrReporter, input, kvWriter);
    }
    done();
}

Example 7 with KeyValueWriter

use of org.apache.tez.runtime.library.api.KeyValueWriter in project tez by apache.

the class MROutput method getWriter.

/**
 * Get a key value write to write Map Reduce compatible output
 */
@Override
public KeyValueWriter getWriter() throws IOException {
    return new KeyValueWriter() {

        private final boolean useNewWriter = useNewApi;

        @SuppressWarnings("unchecked")
        @Override
        public void write(Object key, Object value) throws IOException {
            if (useNewWriter) {
                try {
                    newRecordWriter.write(key, value);
                } catch (InterruptedException e) {
                    Thread.currentThread().interrupt();
                    throw new IOInterruptedException("Interrupted while writing next key-value", e);
                }
            } else {
                oldRecordWriter.write(key, value);
            }
            outputRecordCounter.increment(1);
            getContext().notifyProgress();
        }
    };
}