Examples with Counter org.apache.flink.metrics.Counter used on opensource projects

Example 31 with Counter

use of org.apache.flink.metrics.Counter in project flink by apache.

the class AllReduceDriver method run.

@Override
public void run() throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("AllReduce preprocessing done. Running Reducer code."));
    }
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final ReduceFunction<T> stub = this.taskContext.getStub();
    final MutableObjectIterator<T> input = this.input;
    final TypeSerializer<T> serializer = this.serializer;
    final Collector<T> collector = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    T val1;
    if ((val1 = input.next()) == null) {
        return;
    }
    numRecordsIn.inc();
    if (objectReuseEnabled) {
        // We only need two objects. The first reference stores results and is
        // eventually collected. New values are read into the second.
        T val2 = serializer.createInstance();
        T value = val1;
        while (running && (val2 = input.next(val2)) != null) {
            numRecordsIn.inc();
            value = stub.reduce(value, val2);
            // by the user, so swap the reuse objects,
            if (value == val2) {
                T tmp = val1;
                val1 = val2;
                val2 = tmp;
            }
        }
        collector.collect(value);
    } else {
        T val2;
        while (running && (val2 = input.next()) != null) {
            numRecordsIn.inc();
            val1 = stub.reduce(val1, val2);
        }
        collector.collect(val1);
    }
}

Example 32 with Counter

use of org.apache.flink.metrics.Counter in project flink by apache.

the class CoGroupDriver method prepare.

@Override
public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    if (config.getDriverStrategy() != DriverStrategy.CO_GROUP) {
        throw new Exception("Unrecognized driver strategy for CoGoup driver: " + config.getDriverStrategy().name());
    }
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
    final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
    // get the key positions and types
    final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
    final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
    final TypeComparator<IT1> groupComparator1 = this.taskContext.getDriverComparator(0);
    final TypeComparator<IT2> groupComparator2 = this.taskContext.getDriverComparator(1);
    final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
    if (pairComparatorFactory == null) {
        throw new Exception("Missing pair comparator factory for CoGroup driver");
    }
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    this.objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (LOG.isDebugEnabled()) {
        LOG.debug("CoGroupDriver object reuse: " + (this.objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
    }
    if (objectReuseEnabled) {
        // create CoGropuTaskIterator according to provided local strategy.
        this.coGroupIterator = new ReusingSortMergeCoGroupIterator<IT1, IT2>(in1, in2, serializer1, groupComparator1, serializer2, groupComparator2, pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
    } else {
        // create CoGropuTaskIterator according to provided local strategy.
        this.coGroupIterator = new NonReusingSortMergeCoGroupIterator<IT1, IT2>(in1, in2, serializer1, groupComparator1, serializer2, groupComparator2, pairComparatorFactory.createComparator12(groupComparator1, groupComparator2));
    }
    // open CoGroupTaskIterator - this triggers the sorting and blocks until the iterator is ready
    this.coGroupIterator.open();
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("CoGroup task iterator ready."));
    }
}

Example 33 with Counter

use of org.apache.flink.metrics.Counter in project flink by apache.

the class CoGroupDriver method run.

@Override
public void run() throws Exception {
    final Counter numRecordsOut = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final CoGroupFunction<IT1, IT2, OT> coGroupStub = this.taskContext.getStub();
    final Collector<OT> collector = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    final CoGroupTaskIterator<IT1, IT2> coGroupIterator = this.coGroupIterator;
    while (this.running && coGroupIterator.next()) {
        coGroupStub.coGroup(coGroupIterator.getValues1(), coGroupIterator.getValues2(), collector);
    }
}

Example 34 with Counter

use of org.apache.flink.metrics.Counter in project flink by apache.

the class JoinDriver method run.

@Override
public void run() throws Exception {
    final Counter numRecordsOut = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final FlatJoinFunction<IT1, IT2, OT> joinStub = this.taskContext.getStub();
    final Collector<OT> collector = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    final JoinTaskIterator<IT1, IT2, OT> joinIterator = this.joinIterator;
    while (this.running && joinIterator.callWithNextKey(joinStub, collector)) ;
}

Example 35 with Counter

use of org.apache.flink.metrics.Counter in project flink by apache.

the class MapPartitionDriver method run.

@Override
public void run() throws Exception {
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    // cache references on the stack
    final MutableObjectIterator<IT> input = new CountingMutableObjectIterator<>(this.taskContext.<IT>getInput(0), numRecordsIn);
    final MapPartitionFunction<IT, OT> function = this.taskContext.getStub();
    final Collector<OT> output = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    if (objectReuseEnabled) {
        final ReusingMutableToRegularIteratorWrapper<IT> inIter = new ReusingMutableToRegularIteratorWrapper<IT>(input, this.taskContext.<IT>getInputSerializer(0).getSerializer());
        function.mapPartition(inIter, output);
    } else {
        final NonReusingMutableToRegularIteratorWrapper<IT> inIter = new NonReusingMutableToRegularIteratorWrapper<IT>(input, this.taskContext.<IT>getInputSerializer(0).getSerializer());
        function.mapPartition(inIter, output);
    }
}