Examples with Buffer org.apache.flink.runtime.io.network.buffer.Buffer used on opensource projects

Example 46 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class SingleInputGateTest method testGetCompressedBuffer.

/**
 * Tests that the compressed buffer will be decompressed after calling {@link
 * SingleInputGate#getNext()}.
 */
@Test
public void testGetCompressedBuffer() throws Exception {
    int bufferSize = 1024;
    String compressionCodec = "LZ4";
    BufferCompressor compressor = new BufferCompressor(bufferSize, compressionCodec);
    BufferDecompressor decompressor = new BufferDecompressor(bufferSize, compressionCodec);
    try (SingleInputGate inputGate = new SingleInputGateBuilder().setBufferDecompressor(decompressor).build()) {
        TestInputChannel inputChannel = new TestInputChannel(inputGate, 0);
        MemorySegment segment = MemorySegmentFactory.allocateUnpooledSegment(bufferSize);
        for (int i = 0; i < bufferSize; i += 8) {
            segment.putLongLittleEndian(i, i);
        }
        Buffer uncompressedBuffer = new NetworkBuffer(segment, FreeingBufferRecycler.INSTANCE);
        uncompressedBuffer.setSize(bufferSize);
        Buffer compressedBuffer = compressor.compressToOriginalBuffer(uncompressedBuffer);
        assertTrue(compressedBuffer.isCompressed());
        inputChannel.read(compressedBuffer);
        inputGate.setInputChannels(inputChannel);
        inputGate.notifyChannelNonEmpty(inputChannel);
        Optional<BufferOrEvent> bufferOrEvent = inputGate.getNext();
        assertTrue(bufferOrEvent.isPresent());
        assertTrue(bufferOrEvent.get().isBuffer());
        ByteBuffer buffer = bufferOrEvent.get().getBuffer().getNioBufferReadable().order(ByteOrder.LITTLE_ENDIAN);
        for (int i = 0; i < bufferSize; i += 8) {
            assertEquals(i, buffer.getLong());
        }
    }
}

Example 47 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class PartitionRequestQueueTest method testNotifyNewBufferSize.

@Test
public void testNotifyNewBufferSize() throws Exception {
    // given: Result partition and the reader for subpartition 0.
    ResultPartition parent = createResultPartition();
    BufferAvailabilityListener bufferAvailabilityListener = new NoOpBufferAvailablityListener();
    ResultSubpartitionView view = parent.createSubpartitionView(0, bufferAvailabilityListener);
    ResultPartitionProvider partitionProvider = (partitionId, index, availabilityListener) -> view;
    InputChannelID receiverId = new InputChannelID();
    PartitionRequestQueue queue = new PartitionRequestQueue();
    CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
    EmbeddedChannel channel = new EmbeddedChannel(queue);
    reader.requestSubpartitionView(partitionProvider, new ResultPartitionID(), 0);
    queue.notifyReaderCreated(reader);
    // when: New buffer size received.
    queue.notifyNewBufferSize(receiverId, 65);
    // and: New records emit.
    parent.emitRecord(ByteBuffer.allocate(128), 0);
    parent.emitRecord(ByteBuffer.allocate(10), 0);
    parent.emitRecord(ByteBuffer.allocate(60), 0);
    reader.notifyDataAvailable();
    channel.runPendingTasks();
    // then: Buffers of received size will be in outbound channel.
    Object data1 = channel.readOutbound();
    // The size can not be less than the first record in buffer.
    assertEquals(128, ((NettyMessage.BufferResponse) data1).buffer.getSize());
    Object data2 = channel.readOutbound();
    // The size should shrink up to notified buffer size.
    assertEquals(65, ((NettyMessage.BufferResponse) data2).buffer.getSize());
}

Example 48 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class PartitionRequestQueueTest method testEnqueueReaderByNotifyingBufferAndCredit.

/**
 * Tests {@link PartitionRequestQueue#enqueueAvailableReader(NetworkSequenceViewReader)},
 * verifying the reader would be enqueued in the pipeline iff it has both available credits and
 * buffers.
 */
@Test
public void testEnqueueReaderByNotifyingBufferAndCredit() throws Exception {
    // setup
    final ResultSubpartitionView view = new DefaultBufferResultSubpartitionView(10);
    ResultPartitionProvider partitionProvider = (partitionId, index, availabilityListener) -> view;
    final InputChannelID receiverId = new InputChannelID();
    final PartitionRequestQueue queue = new PartitionRequestQueue();
    final CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
    final EmbeddedChannel channel = new EmbeddedChannel(queue);
    reader.addCredit(-2);
    reader.requestSubpartitionView(partitionProvider, new ResultPartitionID(), 0);
    queue.notifyReaderCreated(reader);
    // block the channel so that we see an intermediate state in the test
    ByteBuf channelBlockingBuffer = blockChannel(channel);
    assertNull(channel.readOutbound());
    // Notify available buffers to trigger enqueue the reader
    final int notifyNumBuffers = 5;
    for (int i = 0; i < notifyNumBuffers; i++) {
        reader.notifyDataAvailable();
    }
    channel.runPendingTasks();
    // the reader is not enqueued in the pipeline because no credits are available
    // -> it should still have the same number of pending buffers
    assertEquals(0, queue.getAvailableReaders().size());
    assertTrue(reader.hasBuffersAvailable().isAvailable());
    assertFalse(reader.isRegisteredAsAvailable());
    assertEquals(0, reader.getNumCreditsAvailable());
    // Notify available credits to trigger enqueue the reader again
    final int notifyNumCredits = 3;
    for (int i = 1; i <= notifyNumCredits; i++) {
        queue.addCreditOrResumeConsumption(receiverId, viewReader -> viewReader.addCredit(1));
        // the reader is enqueued in the pipeline because it has both available buffers and
        // credits
        // since the channel is blocked though, we will not process anything and only enqueue
        // the
        // reader once
        assertTrue(reader.isRegisteredAsAvailable());
        // contains only (this) one!
        assertThat(queue.getAvailableReaders(), contains(reader));
        assertEquals(i, reader.getNumCreditsAvailable());
        assertTrue(reader.hasBuffersAvailable().isAvailable());
    }
    // Flush the buffer to make the channel writable again and see the final results
    channel.flush();
    assertSame(channelBlockingBuffer, channel.readOutbound());
    assertEquals(0, queue.getAvailableReaders().size());
    assertEquals(0, reader.getNumCreditsAvailable());
    assertTrue(reader.hasBuffersAvailable().isAvailable());
    assertFalse(reader.isRegisteredAsAvailable());
    for (int i = 1; i <= notifyNumCredits; i++) {
        assertThat(channel.readOutbound(), instanceOf(NettyMessage.BufferResponse.class));
    }
    assertNull(channel.readOutbound());
}

Example 49 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class BroadcastRecordWriterTest method testRandomEmitAndBufferRecycling.

/**
 * FLINK-17780: Tests that a shared buffer(or memory segment) of a buffer builder is only freed
 * when all consumers are closed.
 */
@Test
public void testRandomEmitAndBufferRecycling() throws Exception {
    int recordSize = 8;
    int numberOfChannels = 2;
    ResultPartition partition = createResultPartition(2 * recordSize, numberOfChannels);
    BufferPool bufferPool = partition.getBufferPool();
    BroadcastRecordWriter<SerializationTestType> writer = new BroadcastRecordWriter<>(partition, -1, "test");
    // force materialization of both buffers for easier availability tests
    List<Buffer> buffers = Arrays.asList(bufferPool.requestBuffer(), bufferPool.requestBuffer());
    buffers.forEach(Buffer::recycleBuffer);
    assertEquals(3, bufferPool.getNumberOfAvailableMemorySegments());
    // fill first buffer
    writer.broadcastEmit(new IntType(1));
    writer.broadcastEmit(new IntType(2));
    assertEquals(2, bufferPool.getNumberOfAvailableMemorySegments());
    // simulate consumption of first buffer consumer; this should not free buffers
    assertEquals(1, partition.getNumberOfQueuedBuffers(0));
    ResultSubpartitionView view0 = partition.createSubpartitionView(0, new NoOpBufferAvailablityListener());
    closeConsumer(view0, 2 * recordSize);
    assertEquals(2, bufferPool.getNumberOfAvailableMemorySegments());
    // use second buffer
    writer.emit(new IntType(3), 0);
    assertEquals(1, bufferPool.getNumberOfAvailableMemorySegments());
    // fully free first buffer
    assertEquals(1, partition.getNumberOfQueuedBuffers(1));
    ResultSubpartitionView view1 = partition.createSubpartitionView(1, new NoOpBufferAvailablityListener());
    closeConsumer(view1, 2 * recordSize);
    assertEquals(2, bufferPool.getNumberOfAvailableMemorySegments());
}

Example 50 with Buffer

use of org.apache.flink.runtime.io.network.buffer.Buffer in project flink by apache.

the class BufferFileWriterFileSegmentReaderTest method testWriteRead.

@Test
public void testWriteRead() throws IOException, InterruptedException {
    int numBuffers = 1024;
    int currentNumber = 0;
    final int minBufferSize = BUFFER_SIZE / 4;
    // Write buffers filled with ascending numbers...
    for (int i = 0; i < numBuffers; i++) {
        final Buffer buffer = createBuffer();
        int size = getNextMultipleOf(getRandomNumberInRange(minBufferSize, BUFFER_SIZE), 4);
        currentNumber = fillBufferWithAscendingNumbers(buffer, currentNumber, size);
        writer.writeBlock(buffer);
    }
    // Make sure that the writes are finished
    writer.close();
    // Read buffers back in...
    for (int i = 0; i < numBuffers; i++) {
        assertFalse(reader.hasReachedEndOfFile());
        reader.read();
    }
    // Wait for all requests to be finished
    final CountDownLatch sync = new CountDownLatch(1);
    final NotificationListener listener = new NotificationListener() {

        @Override
        public void onNotification() {
            sync.countDown();
        }
    };
    if (reader.registerAllRequestsProcessedListener(listener)) {
        sync.await();
    }
    assertTrue(reader.hasReachedEndOfFile());
    // Verify that the content is the same
    assertEquals("Read less buffers than written.", numBuffers, returnedFileSegments.size());
    currentNumber = 0;
    FileSegment fileSegment;
    ByteBuffer buffer = ByteBuffer.allocate(BUFFER_SIZE);
    while ((fileSegment = returnedFileSegments.poll()) != null) {
        buffer.position(0);
        buffer.limit(fileSegment.getLength());
        fileSegment.getFileChannel().read(buffer, fileSegment.getPosition());
        Buffer buffer1 = new NetworkBuffer(MemorySegmentFactory.wrap(buffer.array()), BUFFER_RECYCLER);
        buffer1.setSize(fileSegment.getLength());
        currentNumber = verifyBufferFilledWithAscendingNumbers(buffer1, currentNumber);
    }
    reader.close();
}