Examples with ChannelWithMeta org.apache.flink.table.runtime.io.ChannelWithMeta used on opensource projects

Example 1 with ChannelWithMeta

use of org.apache.flink.table.runtime.io.ChannelWithMeta in project flink by apache.

the class BinaryHashTable method prepareNextPartition.

private boolean prepareNextPartition() throws IOException {
    // finalize and cleanup the partitions of the current table
    for (final BinaryHashPartition p : this.partitionsBeingBuilt) {
        p.finalizeProbePhase(this.internalPool, this.partitionsPending, type.needSetProbed());
    }
    this.partitionsBeingBuilt.clear();
    if (this.currentSpilledBuildSide != null) {
        this.currentSpilledBuildSide.getChannel().closeAndDelete();
        this.currentSpilledBuildSide = null;
    }
    if (this.currentSpilledProbeSide != null) {
        this.currentSpilledProbeSide.getChannel().closeAndDelete();
        this.currentSpilledProbeSide = null;
    }
    if (this.partitionsPending.isEmpty()) {
        // no more data
        return false;
    }
    // there are pending partitions
    final BinaryHashPartition p = this.partitionsPending.get(0);
    LOG.info(String.format("Begin to process spilled partition [%d]", p.getPartitionNumber()));
    if (p.probeSideRecordCounter == 0) {
        // unprobed spilled partitions are only re-processed for a build-side outer join;
        // there is no need to create a hash table since there are no probe-side records
        this.currentSpilledBuildSide = createInputView(p.getBuildSideChannel().getChannelID(), p.getBuildSideBlockCount(), p.getLastSegmentLimit());
        this.buildIterator = new WrappedRowIterator<>(new BinaryRowChannelInputViewIterator(currentSpilledBuildSide, this.binaryBuildSideSerializer), binaryBuildSideSerializer.createInstance());
        this.partitionsPending.remove(0);
        return true;
    }
    this.probeMatchedPhase = true;
    this.buildIterVisited = false;
    // build the next table; memory must be allocated after this call
    buildTableFromSpilledPartition(p);
    // set the probe side
    ChannelWithMeta channelWithMeta = new ChannelWithMeta(p.probeSideBuffer.getChannel().getChannelID(), p.probeSideBuffer.getBlockCount(), p.probeNumBytesInLastSeg);
    this.currentSpilledProbeSide = FileChannelUtil.createInputView(ioManager, channelWithMeta, new ArrayList<>(), compressionEnable, compressionCodecFactory, compressionBlockSize, segmentSize);
    ChannelReaderInputViewIterator<BinaryRowData> probeReader = new ChannelReaderInputViewIterator(this.currentSpilledProbeSide, new ArrayList<>(), this.binaryProbeSideSerializer);
    this.probeIterator.set(probeReader);
    this.probeIterator.setReuse(binaryProbeSideSerializer.createInstance());
    // unregister the pending partition
    this.partitionsPending.remove(0);
    this.currentRecursionDepth = p.getRecursionLevel() + 1;
    // recursively get the next
    return nextMatching();
}

Example 2 with ChannelWithMeta

use of org.apache.flink.table.runtime.io.ChannelWithMeta in project flink by apache.

the class AbstractBinaryExternalMerger method getMergingIterator.

/**
 * Returns an iterator that iterates over the merged result from all given channels.
 *
 * @param channelIDs The channels that are to be merged and returned.
 * @return An iterator over the merged records of the input channels.
 * @throws IOException Thrown, if the readers encounter an I/O problem.
 */
public BinaryMergeIterator<Entry> getMergingIterator(List<ChannelWithMeta> channelIDs, List<FileIOChannel> openChannels) throws IOException {
    // create one iterator per channel id
    if (LOG.isDebugEnabled()) {
        LOG.debug("Performing merge of " + channelIDs.size() + " sorted streams.");
    }
    final List<MutableObjectIterator<Entry>> iterators = new ArrayList<>(channelIDs.size() + 1);
    for (ChannelWithMeta channel : channelIDs) {
        AbstractChannelReaderInputView view = FileChannelUtil.createInputView(ioManager, channel, openChannels, compressionEnable, compressionCodecFactory, compressionBlockSize, pageSize);
        iterators.add(channelReaderInputViewIterator(view));
    }
    return new BinaryMergeIterator<>(iterators, mergeReusedEntries(channelIDs.size()), mergeComparator());
}

Example 3 with ChannelWithMeta

use of org.apache.flink.table.runtime.io.ChannelWithMeta in project flink by apache.

the class AbstractBinaryExternalMerger method mergeChannels.

/**
 * Merges the sorted runs described by the given Channel IDs into a single sorted run.
 *
 * @param channelIDs The IDs of the runs' channels.
 * @return The ID and number of blocks of the channel that describes the merged run.
 */
private ChannelWithMeta mergeChannels(List<ChannelWithMeta> channelIDs) throws IOException {
    // the list with the target iterators
    List<FileIOChannel> openChannels = new ArrayList<>(channelIDs.size());
    final BinaryMergeIterator<Entry> mergeIterator = getMergingIterator(channelIDs, openChannels);
    // create a new channel writer
    final FileIOChannel.ID mergedChannelID = ioManager.createChannel();
    channelManager.addChannel(mergedChannelID);
    AbstractChannelWriterOutputView output = null;
    int numBytesInLastBlock;
    int numBlocksWritten;
    try {
        output = FileChannelUtil.createOutputView(ioManager, mergedChannelID, compressionEnable, compressionCodecFactory, compressionBlockSize, pageSize);
        writeMergingOutput(mergeIterator, output);
        numBytesInLastBlock = output.close();
        numBlocksWritten = output.getBlockCount();
    } catch (IOException e) {
        if (output != null) {
            output.close();
            output.getChannel().deleteChannel();
        }
        throw e;
    }
    // remove, close and delete channels
    for (FileIOChannel channel : openChannels) {
        channelManager.removeChannel(channel.getChannelID());
        try {
            channel.closeAndDelete();
        } catch (Throwable ignored) {
        }
    }
    return new ChannelWithMeta(mergedChannelID, numBlocksWritten, numBytesInLastBlock);
}

Example 4 with ChannelWithMeta

use of org.apache.flink.table.runtime.io.ChannelWithMeta in project flink by apache.

the class BufferedKVExternalSorter method sortAndSpill.

public void sortAndSpill(ArrayList<MemorySegment> recordBufferSegments, long numElements, MemorySegmentPool pool) throws IOException {
    // 1. sort buffer
    BinaryKVInMemorySortBuffer buffer = BinaryKVInMemorySortBuffer.createBuffer(nKeyComputer, keySerializer, valueSerializer, comparator, recordBufferSegments, numElements, pool);
    this.sorter.sort(buffer);
    // 2. spill
    FileIOChannel.ID channel = enumerator.next();
    channelManager.addChannel(channel);
    AbstractChannelWriterOutputView output = null;
    int bytesInLastBuffer;
    int blockCount;
    try {
        numSpillFiles++;
        output = FileChannelUtil.createOutputView(ioManager, channel, compressionEnable, compressionCodecFactory, compressionBlockSize, pageSize);
        buffer.writeToOutput(output);
        spillInBytes += output.getNumBytes();
        spillInCompressedBytes += output.getNumCompressedBytes();
        bytesInLastBuffer = output.close();
        blockCount = output.getBlockCount();
        LOG.info("here spill the {}th kv external buffer data with {} bytes and {} compressed bytes", numSpillFiles, spillInBytes, spillInCompressedBytes);
    } catch (IOException e) {
        if (output != null) {
            output.close();
            output.getChannel().deleteChannel();
        }
        throw e;
    }
    channelIDs.add(new ChannelWithMeta(channel, blockCount, bytesInLastBuffer));
}

Example 5 with ChannelWithMeta

use of org.apache.flink.table.runtime.io.ChannelWithMeta in project flink by apache.

the class ResettableExternalBuffer method clearChannels.

private void clearChannels() {
    for (ChannelWithMeta meta : spilledChannelIDs) {
        final File f = new File(meta.getChannel().getPath());
        if (f.exists()) {
            f.delete();
        }
    }
    spilledChannelIDs.clear();
    spillSize = 0;
    spilledChannelRowOffsets.clear();
}