Examples with CountingCollector org.apache.flink.runtime.operators.util.metrics.CountingCollector used on opensource projects

Example 6 with CountingCollector

use of org.apache.flink.runtime.operators.util.metrics.CountingCollector in project flink by apache.

the class ReduceDriver method run.

@Override
public void run() throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("Reducer preprocessing done. Running Reducer code."));
    }
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    // cache references on the stack
    final MutableObjectIterator<T> input = this.input;
    final TypeSerializer<T> serializer = this.serializer;
    final TypeComparator<T> comparator = this.comparator;
    final ReduceFunction<T> function = this.taskContext.getStub();
    final Collector<T> output = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    if (objectReuseEnabled) {
        // We only need two objects. The first reference stores results and is
        // eventually collected. New values are read into the second.
        //
        // The output value must have the same key fields as the input values.
        T reuse1 = input.next();
        T reuse2 = serializer.createInstance();
        T value = reuse1;
        // iterate over key groups
        while (this.running && value != null) {
            numRecordsIn.inc();
            comparator.setReference(value);
            // iterate within a key group
            while ((reuse2 = input.next(reuse2)) != null) {
                numRecordsIn.inc();
                if (comparator.equalToReference(reuse2)) {
                    // same group, reduce
                    value = function.reduce(value, reuse2);
                    // by the user, so swap the reuse objects
                    if (value == reuse2) {
                        T tmp = reuse1;
                        reuse1 = reuse2;
                        reuse2 = tmp;
                    }
                } else {
                    // new key group
                    break;
                }
            }
            output.collect(value);
            // swap the value from the new key group into the first object
            T tmp = reuse1;
            reuse1 = reuse2;
            reuse2 = tmp;
            value = reuse1;
        }
    } else {
        T value = input.next();
        // iterate over key groups
        while (this.running && value != null) {
            numRecordsIn.inc();
            comparator.setReference(value);
            T res = value;
            // iterate within a key group
            while ((value = input.next()) != null) {
                numRecordsIn.inc();
                if (comparator.equalToReference(value)) {
                    // same group, reduce
                    res = function.reduce(res, value);
                } else {
                    // new key group
                    break;
                }
            }
            output.collect(res);
        }
    }
}

Example 7 with CountingCollector

use of org.apache.flink.runtime.operators.util.metrics.CountingCollector in project flink by apache.

the class UnionWithTempOperator method run.

@Override
public void run() throws Exception {
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final Collector<T> output = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    T reuse = this.taskContext.<T>getInputSerializer(STREAMED_INPUT).getSerializer().createInstance();
    T record;
    final MutableObjectIterator<T> input = this.taskContext.getInput(STREAMED_INPUT);
    while (this.running && ((record = input.next(reuse)) != null)) {
        numRecordsIn.inc();
        output.collect(record);
    }
    final MutableObjectIterator<T> cache = this.taskContext.getInput(CACHED_INPUT);
    while (this.running && ((record = cache.next(reuse)) != null)) {
        numRecordsIn.inc();
        output.collect(record);
    }
}

Example 8 with CountingCollector

use of org.apache.flink.runtime.operators.util.metrics.CountingCollector in project flink by apache.

the class NoOpDriver method run.

@Override
public void run() throws Exception {
    // cache references on the stack
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final MutableObjectIterator<T> input = this.taskContext.getInput(0);
    final Collector<T> output = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    if (objectReuseEnabled) {
        T record = this.taskContext.<T>getInputSerializer(0).getSerializer().createInstance();
        while (this.running && ((record = input.next(record)) != null)) {
            numRecordsIn.inc();
            output.collect(record);
        }
    } else {
        T record;
        while (this.running && ((record = input.next()) != null)) {
            numRecordsIn.inc();
            output.collect(record);
        }
    }
}

Example 9 with CountingCollector

use of org.apache.flink.runtime.operators.util.metrics.CountingCollector in project flink by apache.

the class AbstractOuterJoinDriver method run.

@Override
public void run() throws Exception {
    final Counter numRecordsOut = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final FlatJoinFunction<IT1, IT2, OT> joinStub = this.taskContext.getStub();
    final Collector<OT> collector = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    final JoinTaskIterator<IT1, IT2, OT> outerJoinIterator = this.outerJoinIterator;
    while (this.running && outerJoinIterator.callWithNextKey(joinStub, collector)) ;
}

Example 10 with CountingCollector

use of org.apache.flink.runtime.operators.util.metrics.CountingCollector in project flink by apache.

the class AllGroupCombineDriver method run.

@Override
public void run() throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug("AllGroupCombine starting.");
    }
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final TypeSerializerFactory<IN> serializerFactory = this.taskContext.getInputSerializer(0);
    TypeSerializer<IN> serializer = serializerFactory.getSerializer();
    final MutableObjectIterator<IN> in = new CountingMutableObjectIterator<>(this.taskContext.<IN>getInput(0), numRecordsIn);
    final GroupCombineFunction<IN, OUT> reducer = this.taskContext.getStub();
    final Collector<OUT> output = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    if (objectReuseEnabled) {
        final ReusingMutableToRegularIteratorWrapper<IN> inIter = new ReusingMutableToRegularIteratorWrapper<IN>(in, serializer);
        if (inIter.hasNext()) {
            reducer.combine(inIter, output);
        }
    } else {
        final NonReusingMutableToRegularIteratorWrapper<IN> inIter = new NonReusingMutableToRegularIteratorWrapper<IN>(in, serializer);
        if (inIter.hasNext()) {
            reducer.combine(inIter, output);
        }
    }
}