Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 81 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class LocalInputChannelTest method testCheckpointingInflightData.

@Test
public void testCheckpointingInflightData() throws Exception {
    SingleInputGate inputGate = new SingleInputGateBuilder().build();
    PipelinedResultPartition parent = (PipelinedResultPartition) PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED, NoOpFileChannelManager.INSTANCE);
    ResultSubpartition subpartition = parent.getAllPartitions()[0];
    ResultSubpartitionView subpartitionView = subpartition.createReadView(() -> {
    });
    TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(subpartitionView);
    final RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    LocalInputChannel channel = createLocalInputChannel(inputGate, partitionManager, 0, 0, b -> b.setStateWriter(stateWriter));
    inputGate.setInputChannels(channel);
    channel.requestSubpartition();
    final CheckpointStorageLocationReference location = getDefault();
    CheckpointOptions options = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, location);
    stateWriter.start(0, options);
    final CheckpointBarrier barrier = new CheckpointBarrier(0, 123L, options);
    channel.checkpointStarted(barrier);
    // add 1 buffer before barrier and 1 buffer afterwards. Only the first buffer should be
    // written.
    subpartition.add(createFilledFinishedBufferConsumer(1));
    assertTrue(channel.getNextBuffer().isPresent());
    subpartition.add(EventSerializer.toBufferConsumer(barrier, true));
    assertTrue(channel.getNextBuffer().isPresent());
    subpartition.add(createFilledFinishedBufferConsumer(2));
    assertTrue(channel.getNextBuffer().isPresent());
    assertArrayEquals(stateWriter.getAddedInput().get(channel.getChannelInfo()).stream().mapToInt(Buffer::getSize).toArray(), new int[] { 1 });
}

Example 82 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class NettyMessageClientSideSerializationTest method decompress.

private Buffer decompress(Buffer buffer) {
    MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(BUFFER_SIZE);
    Buffer compressedBuffer = new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE);
    buffer.asByteBuf().readBytes(compressedBuffer.asByteBuf(), buffer.readableBytes());
    compressedBuffer.setCompressed(true);
    return DECOMPRESSOR.decompressToOriginalBuffer(compressedBuffer);
}

Example 83 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class NettyMessageClientSideSerializationTest method testBufferResponse.

private void testBufferResponse(boolean testReadOnlyBuffer, boolean testCompressedBuffer) {
    checkArgument(!(testReadOnlyBuffer & testCompressedBuffer), "There are no cases with both readonly slice and compression.");
    NetworkBuffer buffer = new NetworkBuffer(MemorySegmentFactory.allocateUnpooledSegment(BUFFER_SIZE), FreeingBufferRecycler.INSTANCE);
    for (int i = 0; i < BUFFER_SIZE; i += 8) {
        buffer.writeLong(i);
    }
    Buffer testBuffer = buffer;
    if (testReadOnlyBuffer) {
        testBuffer = buffer.readOnlySlice();
    } else if (testCompressedBuffer) {
        testBuffer = COMPRESSOR.compressToOriginalBuffer(buffer);
    }
    BufferResponse expected = new BufferResponse(testBuffer, random.nextInt(Integer.MAX_VALUE), inputChannelId, random.nextInt(Integer.MAX_VALUE));
    BufferResponse actual = encodeAndDecode(expected, channel);
    assertTrue(buffer.isRecycled());
    assertTrue(testBuffer.isRecycled());
    assertNotNull("The request input channel should always have available buffers in this test.", actual.getBuffer());
    Buffer decodedBuffer = actual.getBuffer();
    if (testCompressedBuffer) {
        assertTrue(actual.isCompressed);
        decodedBuffer = decompress(decodedBuffer);
    }
    verifyBufferResponseHeader(expected, actual);
    assertEquals(BUFFER_SIZE, decodedBuffer.readableBytes());
    for (int i = 0; i < BUFFER_SIZE; i += 8) {
        assertEquals(i, decodedBuffer.asByteBuf().readLong());
    }
    // Release the received message.
    actual.releaseBuffer();
    if (testCompressedBuffer) {
        decodedBuffer.recycleBuffer();
    }
    assertTrue(actual.getBuffer().isRecycled());
}

Example 84 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class PartitionRequestClientHandler method decodeBufferOrEvent.

private boolean decodeBufferOrEvent(RemoteInputChannel inputChannel, NettyMessage.BufferResponse bufferOrEvent, boolean isStagedBuffer) throws Throwable {
    boolean releaseNettyBuffer = true;
    try {
        if (bufferOrEvent.isBuffer()) {
            // IndexOutOfBoundsException.
            if (bufferOrEvent.getSize() == 0) {
                inputChannel.onEmptyBuffer(bufferOrEvent.sequenceNumber);
                return true;
            }
            BufferProvider bufferProvider = inputChannel.getBufferProvider();
            if (bufferProvider == null) {
                // receiver has been cancelled/failed
                cancelRequestFor(bufferOrEvent.receiverId);
                return isStagedBuffer;
            }
            while (true) {
                Buffer buffer = bufferProvider.requestBuffer();
                if (buffer != null) {
                    buffer.setSize(bufferOrEvent.getSize());
                    bufferOrEvent.getNettyBuffer().readBytes(buffer.getNioBuffer());
                    inputChannel.onBuffer(buffer, bufferOrEvent.sequenceNumber);
                    return true;
                } else if (bufferListener.waitForBuffer(bufferProvider, bufferOrEvent)) {
                    releaseNettyBuffer = false;
                    return false;
                } else if (bufferProvider.isDestroyed()) {
                    return isStagedBuffer;
                }
            }
        } else {
            // ---- Event -------------------------------------------------
            // TODO We can just keep the serialized data in the Netty buffer and release it later at the reader
            byte[] byteArray = new byte[bufferOrEvent.getSize()];
            bufferOrEvent.getNettyBuffer().readBytes(byteArray);
            MemorySegment memSeg = MemorySegmentFactory.wrap(byteArray);
            Buffer buffer = new Buffer(memSeg, FreeingBufferRecycler.INSTANCE, false);
            inputChannel.onBuffer(buffer, bufferOrEvent.sequenceNumber);
            return true;
        }
    } finally {
        if (releaseNettyBuffer) {
            bufferOrEvent.releaseBuffer();
        }
    }
}

Example 85 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class EventSerializer method toBuffer.

// ------------------------------------------------------------------------
// Buffer helpers
// ------------------------------------------------------------------------
public static Buffer toBuffer(AbstractEvent event) throws IOException {
    final ByteBuffer serializedEvent = EventSerializer.toSerializedEvent(event);
    MemorySegment data = MemorySegmentFactory.wrap(serializedEvent.array());
    final Buffer buffer = new Buffer(data, FreeingBufferRecycler.INSTANCE, false);
    buffer.setSize(serializedEvent.remaining());
    return buffer;
}