Examples with SpillingResettableMutableObjectIterator org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator used on opensource projects

Example 1 with SpillingResettableMutableObjectIterator

use of org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator in project flink by apache.

the class CrossDriver method runBlockedOuterFirst.

private void runBlockedOuterFirst() throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("Running Cross with Block-Nested-Loops: " + "First input is outer (blocking) side, second input is inner (spilling) side."));
    }
    final Counter numRecordsIn = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final MutableObjectIterator<T1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<T1>getInput(0), numRecordsIn);
    final MutableObjectIterator<T2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<T2>getInput(1), numRecordsIn);
    final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
    final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
    final BlockResettableMutableObjectIterator<T1> blockVals = new BlockResettableMutableObjectIterator<T1>(this.memManager, in1, serializer1, this.memPagesForBlockSide, this.taskContext.getContainingTask());
    this.blockIter = blockVals;
    final SpillingResettableMutableObjectIterator<T2> spillVals = new SpillingResettableMutableObjectIterator<T2>(in2, serializer2, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide, this.taskContext.getContainingTask());
    this.spillIter = spillVals;
    final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
    final Collector<OT> collector = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    if (objectReuseEnabled) {
        final T1 val1Reuse = serializer1.createInstance();
        final T2 val2Reuse = serializer2.createInstance();
        T1 val1;
        T2 val2;
        // for all blocks
        do {
            // for all values from the spilling side
            while (this.running && ((val2 = spillVals.next(val2Reuse)) != null)) {
                // for all values in the block
                while ((val1 = blockVals.next(val1Reuse)) != null) {
                    collector.collect(crosser.cross(val1, val2));
                }
                blockVals.reset();
            }
            spillVals.reset();
        } while (this.running && blockVals.nextBlock());
    } else {
        T1 val1;
        T2 val2;
        // for all blocks
        do {
            // for all values from the spilling side
            while (this.running && ((val2 = spillVals.next()) != null)) {
                // for all values in the block
                while ((val1 = blockVals.next()) != null) {
                    collector.collect(crosser.cross(val1, serializer2.copy(val2)));
                }
                blockVals.reset();
            }
            spillVals.reset();
        } while (this.running && blockVals.nextBlock());
    }
}

Example 2 with SpillingResettableMutableObjectIterator

use of org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator in project flink by apache.

the class CrossDriver method runStreamedOuterSecond.

private void runStreamedOuterSecond() throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("Running Cross with Nested-Loops: " + "First input is inner (spilling) side, second input is outer side."));
    }
    final Counter numRecordsIn = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final MutableObjectIterator<T1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<T1>getInput(0), numRecordsIn);
    final MutableObjectIterator<T2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<T2>getInput(1), numRecordsIn);
    final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
    final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
    final SpillingResettableMutableObjectIterator<T1> spillVals = new SpillingResettableMutableObjectIterator<T1>(in1, serializer1, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide, this.taskContext.getContainingTask());
    this.spillIter = spillVals;
    final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
    final Collector<OT> collector = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    if (objectReuseEnabled) {
        final T1 val1Reuse = serializer1.createInstance();
        final T2 val2Reuse = serializer2.createInstance();
        T1 val1;
        T2 val2;
        // for all blocks
        while (this.running && (val2 = in2.next(val2Reuse)) != null) {
            // for all values from the spilling side
            while (this.running && (val1 = spillVals.next(val1Reuse)) != null) {
                collector.collect(crosser.cross(val1, val2));
            // crosser.cross(val1, val2Copy, collector);
            }
            spillVals.reset();
        }
    } else {
        T1 val1;
        T2 val2;
        // for all blocks
        while (this.running && (val2 = in2.next()) != null) {
            // for all values from the spilling side
            while (this.running && (val1 = spillVals.next()) != null) {
                collector.collect(crosser.cross(val1, serializer2.copy(val2)));
            }
            spillVals.reset();
        }
    }
}

Example 3 with SpillingResettableMutableObjectIterator

use of org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator in project flink by apache.

the class BatchTask method initLocalStrategies.

/**
 * NOTE: This method must be invoked after the invocation of {@code #initInputReaders()} and
 * {@code #initInputSerializersAndComparators(int)}!
 */
protected void initLocalStrategies(int numInputs) throws Exception {
    final MemoryManager memMan = getMemoryManager();
    final IOManager ioMan = getIOManager();
    this.localStrategies = new CloseableInputProvider<?>[numInputs];
    this.inputs = new MutableObjectIterator<?>[numInputs];
    this.excludeFromReset = new boolean[numInputs];
    this.inputIsCached = new boolean[numInputs];
    this.inputIsAsyncMaterialized = new boolean[numInputs];
    this.materializationMemory = new int[numInputs];
    // created
    for (int i = 0; i < numInputs; i++) {
        initInputLocalStrategy(i);
    }
    // we do another loop over the inputs, because we want to instantiate all
    // sorters, etc before requesting the first input (as this call may block)
    // we have two types of materialized inputs, and both are replayable (can act as a cache)
    // The first variant materializes in a different thread and hence
    // acts as a pipeline breaker. this one should only be there, if a pipeline breaker is
    // needed.
    // the second variant spills to the side and will not read unless the result is also
    // consumed
    // in a pipelined fashion.
    this.resettableInputs = new SpillingResettableMutableObjectIterator<?>[numInputs];
    this.tempBarriers = new TempBarrier<?>[numInputs];
    for (int i = 0; i < numInputs; i++) {
        final int memoryPages;
        final boolean async = this.config.isInputAsynchronouslyMaterialized(i);
        final boolean cached = this.config.isInputCached(i);
        this.inputIsAsyncMaterialized[i] = async;
        this.inputIsCached[i] = cached;
        if (async || cached) {
            memoryPages = memMan.computeNumberOfPages(this.config.getRelativeInputMaterializationMemory(i));
            if (memoryPages <= 0) {
                throw new Exception("Input marked as materialized/cached, but no memory for materialization provided.");
            }
            this.materializationMemory[i] = memoryPages;
        } else {
            memoryPages = 0;
        }
        if (async) {
            @SuppressWarnings({ "unchecked", "rawtypes" }) TempBarrier<?> barrier = new TempBarrier(this, getInput(i), this.inputSerializers[i], memMan, ioMan, memoryPages, emptyList());
            barrier.startReading();
            this.tempBarriers[i] = barrier;
            this.inputs[i] = null;
        } else if (cached) {
            @SuppressWarnings({ "unchecked", "rawtypes" }) SpillingResettableMutableObjectIterator<?> iter = new SpillingResettableMutableObjectIterator(getInput(i), this.inputSerializers[i].getSerializer(), getMemoryManager(), getIOManager(), memoryPages, this);
            this.resettableInputs[i] = iter;
            this.inputs[i] = iter;
        }
    }
}

Example 4 with SpillingResettableMutableObjectIterator

use of org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator in project flink by apache.

the class CrossDriver method runBlockedOuterSecond.

private void runBlockedOuterSecond() throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("Running Cross with Block-Nested-Loops: " + "First input is inner (spilling) side, second input is outer (blocking) side."));
    }
    final Counter numRecordsIn = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final MutableObjectIterator<T1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<T1>getInput(0), numRecordsIn);
    final MutableObjectIterator<T2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<T2>getInput(1), numRecordsIn);
    final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
    final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
    final SpillingResettableMutableObjectIterator<T1> spillVals = new SpillingResettableMutableObjectIterator<T1>(in1, serializer1, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide, this.taskContext.getContainingTask());
    this.spillIter = spillVals;
    final BlockResettableMutableObjectIterator<T2> blockVals = new BlockResettableMutableObjectIterator<T2>(this.memManager, in2, serializer2, this.memPagesForBlockSide, this.taskContext.getContainingTask());
    this.blockIter = blockVals;
    final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
    final Collector<OT> collector = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    if (objectReuseEnabled) {
        final T1 val1Reuse = serializer1.createInstance();
        final T2 val2Reuse = serializer2.createInstance();
        T1 val1;
        T2 val2;
        // for all blocks
        do {
            // for all values from the spilling side
            while (this.running && ((val1 = spillVals.next(val1Reuse)) != null)) {
                // for all values in the block
                while (this.running && ((val2 = blockVals.next(val2Reuse)) != null)) {
                    collector.collect(crosser.cross(val1, val2));
                }
                blockVals.reset();
            }
            spillVals.reset();
        } while (this.running && blockVals.nextBlock());
    } else {
        T1 val1;
        T2 val2;
        // for all blocks
        do {
            // for all values from the spilling side
            while (this.running && ((val1 = spillVals.next()) != null)) {
                // for all values in the block
                while (this.running && ((val2 = blockVals.next()) != null)) {
                    collector.collect(crosser.cross(serializer1.copy(val1), val2));
                }
                blockVals.reset();
            }
            spillVals.reset();
        } while (this.running && blockVals.nextBlock());
    }
}

Example 5 with SpillingResettableMutableObjectIterator

use of org.apache.flink.runtime.operators.resettable.SpillingResettableMutableObjectIterator in project flink by apache.

the class CrossDriver method runStreamedOuterFirst.

private void runStreamedOuterFirst() throws Exception {
    if (LOG.isDebugEnabled()) {
        LOG.debug(this.taskContext.formatLogString("Running Cross with Nested-Loops: " + "First input is outer side, second input is inner (spilling) side."));
    }
    final Counter numRecordsIn = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsInCounter();
    final Counter numRecordsOut = taskContext.getMetricGroup().getIOMetricGroup().getNumRecordsOutCounter();
    final MutableObjectIterator<T1> in1 = new CountingMutableObjectIterator<>(this.taskContext.<T1>getInput(0), numRecordsIn);
    final MutableObjectIterator<T2> in2 = new CountingMutableObjectIterator<>(this.taskContext.<T2>getInput(1), numRecordsIn);
    final TypeSerializer<T1> serializer1 = this.taskContext.<T1>getInputSerializer(0).getSerializer();
    final TypeSerializer<T2> serializer2 = this.taskContext.<T2>getInputSerializer(1).getSerializer();
    final SpillingResettableMutableObjectIterator<T2> spillVals = new SpillingResettableMutableObjectIterator<T2>(in2, serializer2, this.memManager, this.taskContext.getIOManager(), this.memPagesForSpillingSide, this.taskContext.getContainingTask());
    this.spillIter = spillVals;
    final CrossFunction<T1, T2, OT> crosser = this.taskContext.getStub();
    final Collector<OT> collector = new CountingCollector<>(this.taskContext.getOutputCollector(), numRecordsOut);
    if (objectReuseEnabled) {
        final T1 val1Reuse = serializer1.createInstance();
        final T2 val2Reuse = serializer2.createInstance();
        T1 val1;
        T2 val2;
        // for all blocks
        while (this.running && ((val1 = in1.next(val1Reuse)) != null)) {
            // for all values from the spilling side
            while (this.running && ((val2 = spillVals.next(val2Reuse)) != null)) {
                collector.collect(crosser.cross(val1, val2));
            }
            spillVals.reset();
        }
    } else {
        T1 val1;
        T2 val2;
        // for all blocks
        while (this.running && ((val1 = in1.next()) != null)) {
            // for all values from the spilling side
            while (this.running && ((val2 = spillVals.next()) != null)) {
                collector.collect(crosser.cross(serializer1.copy(val1), val2));
            }
            spillVals.reset();
        }
    }
}