Examples with NetworkBufferPool org.apache.flink.runtime.io.network.buffer.NetworkBufferPool used on opensource projects

Example 41 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method testAvailableBuffersMoreThanRequiredBuffers.

/**
 * Tests to verify the behaviours of recycling floating and exclusive buffers if the number of
 * available buffers is more than required buffers by decreasing the sender's backlog.
 */
@Test
public void testAvailableBuffersMoreThanRequiredBuffers() throws Exception {
    // Setup
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(16, 32);
    final int numFloatingBuffers = 14;
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
    inputGate.setInputChannels(inputChannel);
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = spy(networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers));
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        // Prepare the exclusive and floating buffers to verify recycle logic later
        final Buffer exclusiveBuffer = inputChannel.requestBuffer();
        assertNotNull(exclusiveBuffer);
        final Buffer floatingBuffer = bufferPool.requestBuffer();
        assertNotNull(floatingBuffer);
        verify(bufferPool, times(1)).requestBuffer();
        // Receive the producer's backlog
        inputChannel.onSenderBacklog(12);
        // The channel gets enough floating buffers from local pool
        verify(bufferPool, times(14)).requestBuffer();
        verify(bufferPool, times(0)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 14 buffers available in the channel", 14, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 14 buffers required in the channel", 14, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments());
        // Decrease the backlog to make the number of available buffers more than required
        // buffers
        inputChannel.onSenderBacklog(10);
        // Only the number of required buffers is changed by (backlog + numExclusiveBuffers)
        verify(bufferPool, times(14)).requestBuffer();
        verify(bufferPool, times(0)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 14 buffers available in the channel", 14, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 12 buffers required in the channel", 12, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments());
        // Recycle one exclusive buffer
        exclusiveBuffer.recycleBuffer();
        // Return one extra floating buffer to the local pool because the number of available
        // buffers
        // is more than required buffers
        verify(bufferPool, times(14)).requestBuffer();
        verify(bufferPool, times(0)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 14 buffers available in the channel", 14, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 12 buffers required in the channel", 12, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 1 buffer available in local pool", 1, bufferPool.getNumberOfAvailableMemorySegments());
        // Recycle one floating buffer
        floatingBuffer.recycleBuffer();
        // The floating buffer is returned to local pool directly because the channel is not
        // waiting for
        // floating buffers
        verify(bufferPool, times(14)).requestBuffer();
        verify(bufferPool, times(0)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 14 buffers available in the channel", 14, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 12 buffers required in the channel", 12, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 2 buffers available in local pool", 2, bufferPool.getNumberOfAvailableMemorySegments());
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, null, null, thrown, inputChannel);
    }
}

Example 42 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method testConcurrentGetNextBufferAndRelease.

@Test
public void testConcurrentGetNextBufferAndRelease() throws Exception {
    final int numTotalBuffers = 1_000;
    final int numFloatingBuffers = 998;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(numTotalBuffers, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
    inputGate.setInputChannels(inputChannel);
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    Throwable thrown = null;
    try {
        BufferPool bufferPool = networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        for (int i = 0; i < numTotalBuffers; i++) {
            Buffer buffer = inputChannel.requestBuffer();
            inputChannel.onBuffer(buffer, i, 0);
        }
        final Callable<Void> getNextBufferTask = () -> {
            try {
                for (int i = 0; i < numTotalBuffers; ++i) {
                    Optional<InputChannel.BufferAndAvailability> bufferAndAvailability = inputChannel.getNextBuffer();
                    bufferAndAvailability.ifPresent(buffer -> buffer.buffer().recycleBuffer());
                }
            } catch (Throwable t) {
                if (!inputChannel.isReleased()) {
                    throw new AssertionError("Exceptions are expected here only if the input channel was released", t);
                }
            }
            return null;
        };
        final Callable<Void> releaseTask = () -> {
            inputChannel.releaseAllResources();
            return null;
        };
        submitTasksAndWaitForResults(executor, new Callable[] { getNextBufferTask, releaseTask });
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, executor, null, thrown, inputChannel);
    }
}

Example 43 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method testAvailableBuffersLessThanRequiredBuffers.

/**
 * Tests to verify the behaviours of three different processes if the number of available
 * buffers is less than required buffers.
 *
 * <ol>
 *   <li>Recycle the floating buffer
 *   <li>Recycle the exclusive buffer
 *   <li>Decrease the sender's backlog
 * </ol>
 */
@Test
public void testAvailableBuffersLessThanRequiredBuffers() throws Exception {
    // Setup
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(16, 32);
    final int numFloatingBuffers = 14;
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
    inputGate.setInputChannels(inputChannel);
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = spy(networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers));
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        // Prepare the exclusive and floating buffers to verify recycle logic later
        final Buffer exclusiveBuffer = inputChannel.requestBuffer();
        assertNotNull(exclusiveBuffer);
        final int numRecycleFloatingBuffers = 2;
        final ArrayDeque<Buffer> floatingBufferQueue = new ArrayDeque<>(numRecycleFloatingBuffers);
        for (int i = 0; i < numRecycleFloatingBuffers; i++) {
            Buffer floatingBuffer = bufferPool.requestBuffer();
            assertNotNull(floatingBuffer);
            floatingBufferQueue.add(floatingBuffer);
        }
        verify(bufferPool, times(numRecycleFloatingBuffers)).requestBuffer();
        // Receive the producer's backlog more than the number of available floating buffers
        inputChannel.onSenderBacklog(14);
        // The channel requests (backlog + numExclusiveBuffers) floating buffers from local
        // pool.
        // It does not get enough floating buffers and register as buffer listener
        verify(bufferPool, times(15)).requestBuffer();
        verify(bufferPool, times(1)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 13 buffers available in the channel", 13, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 16 buffers required in the channel", 16, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments());
        assertTrue(inputChannel.isWaitingForFloatingBuffers());
        // Increase the backlog
        inputChannel.onSenderBacklog(16);
        // The channel is already in the status of waiting for buffers and will not request any
        // more
        verify(bufferPool, times(15)).requestBuffer();
        verify(bufferPool, times(1)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 13 buffers available in the channel", 13, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 18 buffers required in the channel", 18, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments());
        assertTrue(inputChannel.isWaitingForFloatingBuffers());
        // Recycle one exclusive buffer
        exclusiveBuffer.recycleBuffer();
        // The exclusive buffer is returned to the channel directly
        verify(bufferPool, times(15)).requestBuffer();
        verify(bufferPool, times(1)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 14 buffers available in the channel", 14, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 18 buffers required in the channel", 18, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments());
        assertTrue(inputChannel.isWaitingForFloatingBuffers());
        // Recycle one floating buffer
        floatingBufferQueue.poll().recycleBuffer();
        // Assign the floating buffer to the listener and the channel is still waiting for more
        // floating buffers
        verify(bufferPool, times(16)).requestBuffer();
        verify(bufferPool, times(2)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 15 buffers available in the channel", 15, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 18 buffers required in the channel", 18, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments());
        assertTrue(inputChannel.isWaitingForFloatingBuffers());
        // Decrease the backlog
        inputChannel.onSenderBacklog(13);
        // Only the number of required buffers is changed by (backlog + numExclusiveBuffers)
        verify(bufferPool, times(16)).requestBuffer();
        verify(bufferPool, times(2)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 15 buffers available in the channel", 15, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 15 buffers required in the channel", 15, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments());
        assertTrue(inputChannel.isWaitingForFloatingBuffers());
        // Recycle one more floating buffer
        floatingBufferQueue.poll().recycleBuffer();
        // Return the floating buffer to the buffer pool and the channel is not waiting for more
        // floating buffers
        verify(bufferPool, times(16)).requestBuffer();
        verify(bufferPool, times(2)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 15 buffers available in the channel", 15, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 15 buffers required in the channel", 15, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 1 buffers available in local pool", 1, bufferPool.getNumberOfAvailableMemorySegments());
        assertFalse(inputChannel.isWaitingForFloatingBuffers());
        // Increase the backlog again
        inputChannel.onSenderBacklog(15);
        // The floating buffer is requested from the buffer pool and the channel is registered
        // as listener again.
        verify(bufferPool, times(18)).requestBuffer();
        verify(bufferPool, times(3)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 16 buffers available in the channel", 16, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 17 buffers required in the channel", 17, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments());
        assertTrue(inputChannel.isWaitingForFloatingBuffers());
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, null, null, thrown, inputChannel);
    }
}

Example 44 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class SingleInputGateTest method testRequestBuffersWithUnknownInputChannel.

/**
 * Tests that input gate requests and assigns network buffers when unknown input channel updates
 * to remote input channel.
 */
@Test
public void testRequestBuffersWithUnknownInputChannel() throws Exception {
    final NettyShuffleEnvironment network = createNettyShuffleEnvironment();
    final SingleInputGate inputGate = createInputGate(network, 1, ResultPartitionType.PIPELINED_BOUNDED);
    int buffersPerChannel = 2;
    int extraNetworkBuffersPerGate = 8;
    try (Closer closer = Closer.create()) {
        closer.register(network::close);
        closer.register(inputGate::close);
        final ResultPartitionID resultPartitionId = new ResultPartitionID();
        InputChannel inputChannel = buildUnknownInputChannel(network, inputGate, resultPartitionId, 0);
        inputGate.setInputChannels(inputChannel);
        inputGate.setup();
        NetworkBufferPool bufferPool = network.getNetworkBufferPool();
        assertEquals(bufferPool.getTotalNumberOfMemorySegments() - 1, bufferPool.getNumberOfAvailableMemorySegments());
        // note: exclusive buffers are not handed out into LocalBufferPool and are thus not
        // counted
        assertEquals(extraNetworkBuffersPerGate, bufferPool.countBuffers());
        // Trigger updates to remote input channel from unknown input channel
        inputGate.updateInputChannel(ResourceID.generate(), createRemoteWithIdAndLocation(resultPartitionId.getPartitionId(), ResourceID.generate()));
        RemoteInputChannel remote = (RemoteInputChannel) inputGate.getInputChannels().get(createSubpartitionInfo(resultPartitionId.getPartitionId()));
        // only the exclusive buffers should be assigned/available now
        assertEquals(buffersPerChannel, remote.getNumberOfAvailableBuffers());
        assertEquals(bufferPool.getTotalNumberOfMemorySegments() - buffersPerChannel - 1, bufferPool.getNumberOfAvailableMemorySegments());
        // note: exclusive buffers are not handed out into LocalBufferPool and are thus not
        // counted
        assertEquals(extraNetworkBuffersPerGate, bufferPool.countBuffers());
    }
}

Example 45 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class SingleInputGateTest method testRequestBuffersWithRemoteInputChannel.

/**
 * Tests that input gate requests and assigns network buffers for remote input channel.
 */
@Test
public void testRequestBuffersWithRemoteInputChannel() throws Exception {
    final NettyShuffleEnvironment network = createNettyShuffleEnvironment();
    final SingleInputGate inputGate = createInputGate(network, 1, ResultPartitionType.PIPELINED_BOUNDED);
    int buffersPerChannel = 2;
    int extraNetworkBuffersPerGate = 8;
    try (Closer closer = Closer.create()) {
        closer.register(network::close);
        closer.register(inputGate::close);
        RemoteInputChannel remote = InputChannelBuilder.newBuilder().setupFromNettyShuffleEnvironment(network).setConnectionManager(new TestingConnectionManager()).buildRemoteChannel(inputGate);
        inputGate.setInputChannels(remote);
        inputGate.setup();
        NetworkBufferPool bufferPool = network.getNetworkBufferPool();
        // only the exclusive buffers should be assigned/available now
        assertEquals(buffersPerChannel, remote.getNumberOfAvailableBuffers());
        assertEquals(bufferPool.getTotalNumberOfMemorySegments() - buffersPerChannel - 1, bufferPool.getNumberOfAvailableMemorySegments());
        // note: exclusive buffers are not handed out into LocalBufferPool and are thus not
        // counted
        assertEquals(extraNetworkBuffersPerGate, bufferPool.countBuffers());
    }
}