Examples with ChannelReaderInputViewIterator org.apache.flink.runtime.io.disk.ChannelReaderInputViewIterator used on opensource projects

Example 6 with ChannelReaderInputViewIterator

use of org.apache.flink.runtime.io.disk.ChannelReaderInputViewIterator in project flink by apache.

the class LongHybridHashTable method prepareNextPartition.

private boolean prepareNextPartition() throws IOException {
    // finalize and cleanup the partitions of the current table
    for (final LongHashPartition p : this.partitionsBeingBuilt) {
        p.finalizeProbePhase(this.partitionsPending);
    }
    this.partitionsBeingBuilt.clear();
    if (this.currentSpilledProbeSide != null) {
        this.currentSpilledProbeSide.getChannel().closeAndDelete();
        this.currentSpilledProbeSide = null;
    }
    if (this.partitionsPending.isEmpty()) {
        // no more data
        return false;
    }
    // there are pending partitions
    final LongHashPartition p = this.partitionsPending.get(0);
    LOG.info(String.format("Begin to process spilled partition [%d]", p.getPartitionNumber()));
    if (p.probeSideRecordCounter == 0) {
        this.partitionsPending.remove(0);
        return prepareNextPartition();
    }
    // build the next table; memory must be allocated after this call
    buildTableFromSpilledPartition(p);
    // set the probe side
    ChannelWithMeta channelWithMeta = new ChannelWithMeta(p.probeSideBuffer.getChannel().getChannelID(), p.probeSideBuffer.getBlockCount(), p.probeNumBytesInLastSeg);
    this.currentSpilledProbeSide = FileChannelUtil.createInputView(ioManager, channelWithMeta, new ArrayList<>(), compressionEnable, compressionCodecFactory, compressionBlockSize, segmentSize);
    ChannelReaderInputViewIterator<BinaryRowData> probeReader = new ChannelReaderInputViewIterator(this.currentSpilledProbeSide, new ArrayList<>(), this.probeSideSerializer);
    this.probeIterator.set(probeReader);
    this.probeIterator.setReuse(probeSideSerializer.createInstance());
    // unregister the pending partition
    this.partitionsPending.remove(0);
    this.currentRecursionDepth = p.getRecursionLevel() + 1;
    // recursively get the next
    return nextMatching();
}