Examples with AbstractChannelWriterOutputView org.apache.flink.runtime.io.disk.iomanager.AbstractChannelWriterOutputView used on opensource projects

Example 1 with AbstractChannelWriterOutputView

use of org.apache.flink.runtime.io.disk.iomanager.AbstractChannelWriterOutputView in project flink by apache.

the class AbstractBinaryExternalMerger method mergeChannels.

/**
 * Merges the sorted runs described by the given Channel IDs into a single sorted run.
 *
 * @param channelIDs The IDs of the runs' channels.
 * @return The ID and number of blocks of the channel that describes the merged run.
 */
private ChannelWithMeta mergeChannels(List<ChannelWithMeta> channelIDs) throws IOException {
    // the list with the target iterators
    List<FileIOChannel> openChannels = new ArrayList<>(channelIDs.size());
    final BinaryMergeIterator<Entry> mergeIterator = getMergingIterator(channelIDs, openChannels);
    // create a new channel writer
    final FileIOChannel.ID mergedChannelID = ioManager.createChannel();
    channelManager.addChannel(mergedChannelID);
    AbstractChannelWriterOutputView output = null;
    int numBytesInLastBlock;
    int numBlocksWritten;
    try {
        output = FileChannelUtil.createOutputView(ioManager, mergedChannelID, compressionEnable, compressionCodecFactory, compressionBlockSize, pageSize);
        writeMergingOutput(mergeIterator, output);
        numBytesInLastBlock = output.close();
        numBlocksWritten = output.getBlockCount();
    } catch (IOException e) {
        if (output != null) {
            output.close();
            output.getChannel().deleteChannel();
        }
        throw e;
    }
    // remove, close and delete channels
    for (FileIOChannel channel : openChannels) {
        channelManager.removeChannel(channel.getChannelID());
        try {
            channel.closeAndDelete();
        } catch (Throwable ignored) {
        }
    }
    return new ChannelWithMeta(mergedChannelID, numBlocksWritten, numBytesInLastBlock);
}

Example 2 with AbstractChannelWriterOutputView

use of org.apache.flink.runtime.io.disk.iomanager.AbstractChannelWriterOutputView in project flink by apache.

the class BufferedKVExternalSorter method sortAndSpill.

public void sortAndSpill(ArrayList<MemorySegment> recordBufferSegments, long numElements, MemorySegmentPool pool) throws IOException {
    // 1. sort buffer
    BinaryKVInMemorySortBuffer buffer = BinaryKVInMemorySortBuffer.createBuffer(nKeyComputer, keySerializer, valueSerializer, comparator, recordBufferSegments, numElements, pool);
    this.sorter.sort(buffer);
    // 2. spill
    FileIOChannel.ID channel = enumerator.next();
    channelManager.addChannel(channel);
    AbstractChannelWriterOutputView output = null;
    int bytesInLastBuffer;
    int blockCount;
    try {
        numSpillFiles++;
        output = FileChannelUtil.createOutputView(ioManager, channel, compressionEnable, compressionCodecFactory, compressionBlockSize, pageSize);
        buffer.writeToOutput(output);
        spillInBytes += output.getNumBytes();
        spillInCompressedBytes += output.getNumCompressedBytes();
        bytesInLastBuffer = output.close();
        blockCount = output.getBlockCount();
        LOG.info("here spill the {}th kv external buffer data with {} bytes and {} compressed bytes", numSpillFiles, spillInBytes, spillInCompressedBytes);
    } catch (IOException e) {
        if (output != null) {
            output.close();
            output.getChannel().deleteChannel();
        }
        throw e;
    }
    channelIDs.add(new ChannelWithMeta(channel, blockCount, bytesInLastBuffer));
}