Examples with CountingMutableObjectIterator org.apache.flink.runtime.operators.util.metrics.CountingMutableObjectIterator used on opensource projects

Example 1 with CountingMutableObjectIterator

use of org.apache.flink.runtime.operators.util.metrics.CountingMutableObjectIterator in project flink by apache.

the class CrossDriver method runBlockedOuterFirst.

private void runBlockedOuterFirst() throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("Running Cross with Block-Nested-Loops: " + "First input is outer (blocking) side, second input is inner (spilling) side."));
    }
    final Counter numRecordsIn = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final MutableObjectIterator<T1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<T1>getInput(0), numRecordsIn);
    final MutableObjectIterator<T2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<T2>getInput(1), numRecordsIn);
    final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
    final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
    final BlockResettableMutableObjectIterator<T1> blockVals = new BlockResettableMutableObjectIterator<T1>(this.memManager, in1, serializer1, this.memPagesForBlockSide, this.taskContext.getContainingTask());
    this.blockIter = blockVals;
    final SpillingResettableMutableObjectIterator<T2> spillVals = new SpillingResettableMutableObjectIterator<T2>(in2, serializer2, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide, this.taskContext.getContainingTask());
    this.spillIter = spillVals;
    final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
    final Collector<OT> collector = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    if (objectReuseEnabled) {
        final T1 val1Reuse = serializer1.createInstance();
        final T2 val2Reuse = serializer2.createInstance();
        T1 val1;
        T2 val2;
        // for all blocks
        do {
            // for all values from the spilling side
            while (this.running && ((val2 = spillVals.next(val2Reuse)) != null)) {
                // for all values in the block
                while ((val1 = blockVals.next(val1Reuse)) != null) {
                    collector.collect(crosser.cross(val1, val2));
                }
                blockVals.reset();
            }
            spillVals.reset();
        } while (this.running && blockVals.nextBlock());
    } else {
        T1 val1;
        T2 val2;
        // for all blocks
        do {
            // for all values from the spilling side
            while (this.running && ((val2 = spillVals.next()) != null)) {
                // for all values in the block
                while ((val1 = blockVals.next()) != null) {
                    collector.collect(crosser.cross(val1, serializer2.copy(val2)));
                }
                blockVals.reset();
            }
            spillVals.reset();
        } while (this.running && blockVals.nextBlock());
    }
}

Example 2 with CountingMutableObjectIterator

use of org.apache.flink.runtime.operators.util.metrics.CountingMutableObjectIterator in project flink by apache.

the class CrossDriver method runStreamedOuterSecond.

private void runStreamedOuterSecond() throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("Running Cross with Nested-Loops: " + "First input is inner (spilling) side, second input is outer side."));
    }
    final Counter numRecordsIn = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final MutableObjectIterator<T1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<T1>getInput(0), numRecordsIn);
    final MutableObjectIterator<T2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<T2>getInput(1), numRecordsIn);
    final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
    final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
    final SpillingResettableMutableObjectIterator<T1> spillVals = new SpillingResettableMutableObjectIterator<T1>(in1, serializer1, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide, this.taskContext.getContainingTask());
    this.spillIter = spillVals;
    final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
    final Collector<OT> collector = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    if (objectReuseEnabled) {
        final T1 val1Reuse = serializer1.createInstance();
        final T2 val2Reuse = serializer2.createInstance();
        T1 val1;
        T2 val2;
        // for all blocks
        while (this.running && (val2 = in2.next(val2Reuse)) != null) {
            // for all values from the spilling side
            while (this.running && (val1 = spillVals.next(val1Reuse)) != null) {
                collector.collect(crosser.cross(val1, val2));
            // crosser.cross(val1, val2Copy, collector);
            }
            spillVals.reset();
        }
    } else {
        T1 val1;
        T2 val2;
        // for all blocks
        while (this.running && (val2 = in2.next()) != null) {
            // for all values from the spilling side
            while (this.running && (val1 = spillVals.next()) != null) {
                collector.collect(crosser.cross(val1, serializer2.copy(val2)));
            }
            spillVals.reset();
        }
    }
}

Example 3 with CountingMutableObjectIterator

use of org.apache.flink.runtime.operators.util.metrics.CountingMutableObjectIterator in project flink by apache.

the class AbstractOuterJoinDriver method prepare.

@Override
public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    // obtain task manager's memory manager and I/O manager
    final MemoryManager memoryManager = this.taskContext.getMemoryManager();
    final IOManager ioManager = this.taskContext.getIOManager();
    // set up memory and I/O parameters
    final double driverMemFraction = config.getRelativeMemoryDriver();
    final DriverStrategy ls = config.getDriverStrategy();
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
    final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
    // get serializers and comparators
    final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
    final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
    final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
    final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
    final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
    if (pairComparatorFactory == null) {
        throw new Exception("Missing pair comparator factory for outer join driver");
    }
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Outer Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
    }
    // create and return outer join iterator according to provided local strategy.
    if (objectReuseEnabled) {
        this.outerJoinIterator = getReusingOuterJoinIterator(ls, in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory, memoryManager, ioManager, driverMemFraction);
    } else {
        this.outerJoinIterator = getNonReusingOuterJoinIterator(ls, in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory, memoryManager, ioManager, driverMemFraction);
    }
    this.outerJoinIterator.open();
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("outer join task iterator ready."));
    }
}

Example 4 with CountingMutableObjectIterator

use of org.apache.flink.runtime.operators.util.metrics.CountingMutableObjectIterator in project flink by apache.

the class JoinDriver method prepare.

@Override
public void prepare() throws Exception {
    final TaskConfig config = this.taskContext.getTaskConfig();
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    // obtain task manager's memory manager and I/O manager
    final MemoryManager memoryManager = this.taskContext.getMemoryManager();
    final IOManager ioManager = this.taskContext.getIOManager();
    // set up memory and I/O parameters
    final double fractionAvailableMemory = config.getRelativeMemoryDriver();
    final int numPages = memoryManager.computeNumberOfPages(fractionAvailableMemory);
    // test minimum memory requirements
    final DriverStrategy ls = config.getDriverStrategy();
    final MutableObjectIterator<IT1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<IT1>getInput(0), numRecordsIn);
    final MutableObjectIterator<IT2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<IT2>getInput(1), numRecordsIn);
    // get the key positions and types
    final TypeSerializer<IT1> serializer1 = this.taskContext.<IT1>getInputSerializer(0).getSerializer();
    final TypeSerializer<IT2> serializer2 = this.taskContext.<IT2>getInputSerializer(1).getSerializer();
    final TypeComparator<IT1> comparator1 = this.taskContext.getDriverComparator(0);
    final TypeComparator<IT2> comparator2 = this.taskContext.getDriverComparator(1);
    final TypePairComparatorFactory<IT1, IT2> pairComparatorFactory = config.getPairComparatorFactory(this.taskContext.getUserCodeClassLoader());
    if (pairComparatorFactory == null) {
        throw new Exception("Missing pair comparator factory for join driver");
    }
    ExecutionConfig executionConfig = taskContext.getExecutionConfig();
    boolean objectReuseEnabled = executionConfig.isObjectReuseEnabled();
    if (LOG.isDebugEnabled()) {
        LOG.debug("Join Driver object reuse: " + (objectReuseEnabled ? "ENABLED" : "DISABLED") + ".");
    }
    boolean hashJoinUseBitMaps = taskContext.getTaskManagerInfo().getConfiguration().getBoolean(AlgorithmOptions.HASH_JOIN_BLOOM_FILTERS);
    // create and return joining iterator according to provided local strategy.
    if (objectReuseEnabled) {
        switch(ls) {
            case INNER_MERGE:
                this.joinIterator = new ReusingMergeInnerJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2), memoryManager, ioManager, numPages, this.taskContext.getContainingTask());
                break;
            case HYBRIDHASH_BUILD_FIRST:
                this.joinIterator = new ReusingBuildFirstHashJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator21(comparator1, comparator2), memoryManager, ioManager, this.taskContext.getContainingTask(), fractionAvailableMemory, false, false, hashJoinUseBitMaps);
                break;
            case HYBRIDHASH_BUILD_SECOND:
                this.joinIterator = new ReusingBuildSecondHashJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2), memoryManager, ioManager, this.taskContext.getContainingTask(), fractionAvailableMemory, false, false, hashJoinUseBitMaps);
                break;
            default:
                throw new Exception("Unsupported driver strategy for join driver: " + ls.name());
        }
    } else {
        switch(ls) {
            case INNER_MERGE:
                this.joinIterator = new NonReusingMergeInnerJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2), memoryManager, ioManager, numPages, this.taskContext.getContainingTask());
                break;
            case HYBRIDHASH_BUILD_FIRST:
                this.joinIterator = new NonReusingBuildFirstHashJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator21(comparator1, comparator2), memoryManager, ioManager, this.taskContext.getContainingTask(), fractionAvailableMemory, false, false, hashJoinUseBitMaps);
                break;
            case HYBRIDHASH_BUILD_SECOND:
                this.joinIterator = new NonReusingBuildSecondHashJoinIterator<>(in1, in2, serializer1, comparator1, serializer2, comparator2, pairComparatorFactory.createComparator12(comparator1, comparator2), memoryManager, ioManager, this.taskContext.getContainingTask(), fractionAvailableMemory, false, false, hashJoinUseBitMaps);
                break;
            default:
                throw new Exception("Unsupported driver strategy for join driver: " + ls.name());
        }
    }
    // open the iterator - this triggers the sorting or hash-table building
    // and blocks until the iterator is ready
    this.joinIterator.open();
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("join task iterator ready."));
    }
}

Example 5 with CountingMutableObjectIterator

use of org.apache.flink.runtime.operators.util.metrics.CountingMutableObjectIterator in project flink by apache.

the class MapPartitionDriver method run.

@Override
public void run() throws Exception {
    final Counter numRecordsIn = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = this.taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    // cache references on the stack
    final MutableObjectIterator<IT> input = new CountingMutableObjectIterator<>(this.taskContext.<IT>getInput(0), numRecordsIn);
    final MapPartitionFunction<IT, OT> function = this.taskContext.getStub();
    final Collector<OT> output = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    if (objectReuseEnabled) {
        final ReusingMutableToRegularIteratorWrapper<IT> inIter = new ReusingMutableToRegularIteratorWrapper<IT>(input, this.taskContext.<IT>getInputSerializer(0).getSerializer());
        function.mapPartition(inIter, output);
    } else {
        final NonReusingMutableToRegularIteratorWrapper<IT> inIter = new NonReusingMutableToRegularIteratorWrapper<IT>(input, this.taskContext.<IT>getInputSerializer(0).getSerializer());
        function.mapPartition(inIter, output);
    }
}