Examples with NetworkBufferPool org.apache.flink.runtime.io.network.buffer.NetworkBufferPool used on opensource projects

Example 11 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class DataBufferTest method testReleaseDataBuffer.

@Test
public void testReleaseDataBuffer() throws Exception {
    int bufferPoolSize = 10;
    int bufferSize = 1024;
    int recordSize = (bufferPoolSize - 1) * bufferSize;
    NetworkBufferPool globalPool = new NetworkBufferPool(bufferPoolSize, bufferSize);
    BufferPool bufferPool = globalPool.createBufferPool(bufferPoolSize, bufferPoolSize);
    DataBuffer dataBuffer = new SortBasedDataBuffer(bufferPool, 1, bufferSize, bufferPoolSize, null);
    dataBuffer.append(ByteBuffer.allocate(recordSize), 0, Buffer.DataType.DATA_BUFFER);
    assertEquals(bufferPoolSize, bufferPool.bestEffortGetNumOfUsedBuffers());
    assertTrue(dataBuffer.hasRemaining());
    assertEquals(1, dataBuffer.numTotalRecords());
    assertEquals(recordSize, dataBuffer.numTotalBytes());
    // should release all data and resources
    dataBuffer.release();
    assertEquals(0, bufferPool.bestEffortGetNumOfUsedBuffers());
    assertTrue(dataBuffer.hasRemaining());
    assertEquals(1, dataBuffer.numTotalRecords());
    assertEquals(recordSize, dataBuffer.numTotalBytes());
}

Example 12 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method testConcurrentRecycleAndRelease2.

/**
 * Tests to verify that there is no race condition with two things running in parallel:
 * recycling exclusive buffers and recycling external buffers to the buffer pool while the
 * recycling of the exclusive buffer triggers recycling a floating buffer (FLINK-9676).
 */
@Test
public void testConcurrentRecycleAndRelease2() throws Exception {
    // Setup
    final int retries = 1_000;
    final int numExclusiveBuffers = 2;
    final int numFloatingBuffers = 2;
    final int numTotalBuffers = numExclusiveBuffers + numFloatingBuffers;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(numTotalBuffers, 32);
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
    inputGate.setInputChannels(inputChannel);
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        final Callable<Void> bufferPoolInteractionsTask = () -> {
            for (int i = 0; i < retries; ++i) {
                try (BufferBuilder bufferBuilder = bufferPool.requestBufferBuilderBlocking()) {
                    Buffer buffer = buildSingleBuffer(bufferBuilder);
                    buffer.recycleBuffer();
                }
            }
            return null;
        };
        final Callable<Void> channelInteractionsTask = () -> {
            ArrayList<Buffer> exclusiveBuffers = new ArrayList<>(numExclusiveBuffers);
            ArrayList<Buffer> floatingBuffers = new ArrayList<>(numExclusiveBuffers);
            try {
                for (int i = 0; i < retries; ++i) {
                    // floating buffers as soon as we take exclusive ones
                    for (int j = 0; j < numTotalBuffers; ++j) {
                        Buffer buffer = inputChannel.requestBuffer();
                        if (buffer == null) {
                            break;
                        } else {
                            // noinspection ObjectEquality
                            if (buffer.getRecycler() == inputChannel.getBufferManager()) {
                                exclusiveBuffers.add(buffer);
                            } else {
                                floatingBuffers.add(buffer);
                            }
                        }
                    }
                    // recycle excess floating buffers (will go back into the channel)
                    floatingBuffers.forEach(Buffer::recycleBuffer);
                    floatingBuffers.clear();
                    assertEquals(numExclusiveBuffers, exclusiveBuffers.size());
                    inputChannel.onSenderBacklog(// trigger subscription to buffer pool
                    0);
                    // note: if we got a floating buffer by increasing the backlog, it
                    // will be released again when recycling the exclusive buffer, if
                    // not, we should release it once we get it
                    exclusiveBuffers.forEach(Buffer::recycleBuffer);
                    exclusiveBuffers.clear();
                }
            } finally {
                inputChannel.releaseAllResources();
            }
            return null;
        };
        // Submit tasks and wait to finish
        submitTasksAndWaitForResults(executor, new Callable[] { bufferPoolInteractionsTask, channelInteractionsTask });
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, executor, null, thrown, inputChannel);
    }
}

Example 13 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testReadBufferResponseWithReleasingOrRemovingChannel.

private void testReadBufferResponseWithReleasingOrRemovingChannel(boolean isRemoved, boolean readBeforeReleasingOrRemoving) throws Exception {
    int bufferSize = 1024;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, bufferSize);
    SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    RemoteInputChannel inputChannel = new InputChannelBuilder().buildRemoteChannel(inputGate);
    inputGate.setInputChannels(inputChannel);
    inputGate.setup();
    CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    EmbeddedChannel embeddedChannel = new EmbeddedChannel(handler);
    handler.addInputChannel(inputChannel);
    try {
        if (!readBeforeReleasingOrRemoving) {
            // Release the channel.
            inputGate.close();
            if (isRemoved) {
                handler.removeInputChannel(inputChannel);
            }
        }
        BufferResponse bufferResponse = createBufferResponse(TestBufferFactory.createBuffer(bufferSize), 0, inputChannel.getInputChannelId(), 1, new NetworkBufferAllocator(handler));
        if (readBeforeReleasingOrRemoving) {
            // Release the channel.
            inputGate.close();
            if (isRemoved) {
                handler.removeInputChannel(inputChannel);
            }
        }
        handler.channelRead(null, bufferResponse);
        assertEquals(0, inputChannel.getNumberOfQueuedBuffers());
        if (!readBeforeReleasingOrRemoving) {
            assertNull(bufferResponse.getBuffer());
        } else {
            assertNotNull(bufferResponse.getBuffer());
            assertTrue(bufferResponse.getBuffer().isRecycled());
        }
        embeddedChannel.runScheduledPendingTasks();
        NettyMessage.CancelPartitionRequest cancelPartitionRequest = embeddedChannel.readOutbound();
        assertNotNull(cancelPartitionRequest);
        assertEquals(inputChannel.getInputChannelId(), cancelPartitionRequest.receiverId);
    } finally {
        releaseResource(inputGate, networkBufferPool);
        embeddedChannel.close();
    }
}

Example 14 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testAnnounceBufferSize.

@Test
public void testAnnounceBufferSize() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = new NettyPartitionRequestClient(channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(2, networkBufferPool);
    final RemoteInputChannel[] inputChannels = new RemoteInputChannel[2];
    inputChannels[0] = createRemoteInputChannel(inputGate, client);
    inputChannels[1] = createRemoteInputChannel(inputGate, client);
    try {
        inputGate.setInputChannels(inputChannels);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannels[0].requestSubpartition();
        inputChannels[1].requestSubpartition();
        channel.readOutbound();
        channel.readOutbound();
        inputGate.announceBufferSize(333);
        channel.runPendingTasks();
        NettyMessage.NewBufferSize readOutbound = channel.readOutbound();
        assertThat(readOutbound, instanceOf(NettyMessage.NewBufferSize.class));
        assertThat(readOutbound.receiverId, is(inputChannels[0].getInputChannelId()));
        assertThat(readOutbound.bufferSize, is(333));
        readOutbound = channel.readOutbound();
        assertThat(readOutbound.receiverId, is(inputChannels[1].getInputChannelId()));
        assertThat(readOutbound.bufferSize, is(333));
    } finally {
        releaseResource(inputGate, networkBufferPool);
        channel.close();
    }
}

Example 15 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testReceiveBacklogAnnouncement.

/**
 * Verifies that {@link NettyMessage.BacklogAnnouncement} can be handled correctly.
 */
@Test
public void testReceiveBacklogAnnouncement() throws Exception {
    int bufferSize = 1024;
    int numBuffers = 10;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(numBuffers, bufferSize);
    SingleInputGate inputGate = new SingleInputGateBuilder().setSegmentProvider(networkBufferPool).build();
    RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, null);
    inputGate.setInputChannels(inputChannel);
    try {
        BufferPool bufferPool = networkBufferPool.createBufferPool(8, 8);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
        handler.addInputChannel(inputChannel);
        assertEquals(2, inputChannel.getNumberOfAvailableBuffers());
        assertEquals(0, inputChannel.unsynchronizedGetFloatingBuffersAvailable());
        int backlog = 5;
        NettyMessage.BacklogAnnouncement announcement = new NettyMessage.BacklogAnnouncement(backlog, inputChannel.getInputChannelId());
        handler.channelRead(null, announcement);
        assertEquals(7, inputChannel.getNumberOfAvailableBuffers());
        assertEquals(7, inputChannel.getNumberOfRequiredBuffers());
        assertEquals(backlog, inputChannel.getSenderBacklog());
        assertEquals(5, inputChannel.unsynchronizedGetFloatingBuffersAvailable());
        backlog = 12;
        announcement = new NettyMessage.BacklogAnnouncement(backlog, inputChannel.getInputChannelId());
        handler.channelRead(null, announcement);
        assertEquals(10, inputChannel.getNumberOfAvailableBuffers());
        assertEquals(14, inputChannel.getNumberOfRequiredBuffers());
        assertEquals(backlog, inputChannel.getSenderBacklog());
        assertEquals(8, inputChannel.unsynchronizedGetFloatingBuffersAvailable());
    } finally {
        releaseResource(inputGate, networkBufferPool);
    }
}