Examples with NetworkBufferPool org.apache.flink.runtime.io.network.buffer.NetworkBufferPool used on opensource projects

Example 36 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method testConcurrentOnSenderBacklogAndRelease.

/**
 * Tests to verify that there is no race condition with two things running in parallel:
 * requesting floating buffers on sender backlog and some other thread releasing the input
 * channel.
 */
@Test
public void testConcurrentOnSenderBacklogAndRelease() throws Exception {
    // Setup
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(130, 32);
    final int numFloatingBuffers = 128;
    final ExecutorService executor = Executors.newFixedThreadPool(2);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        final Callable<Void> requestBufferTask = new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                while (true) {
                    for (int j = 1; j <= numFloatingBuffers; j++) {
                        inputChannel.onSenderBacklog(j);
                    }
                    if (inputChannel.isReleased()) {
                        return null;
                    }
                }
            }
        };
        final Callable<Void> releaseTask = new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                inputChannel.releaseAllResources();
                return null;
            }
        };
        // Submit tasks and wait to finish
        submitTasksAndWaitForResults(executor, new Callable[] { requestBufferTask, releaseTask });
        assertEquals("There should be no buffers available in the channel.", 0, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 130 buffers available in local pool.", 130, bufferPool.getNumberOfAvailableMemorySegments() + networkBufferPool.getNumberOfAvailableMemorySegments());
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, executor, null, thrown, inputChannel);
    }
}

Example 37 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method testFailureInNotifyBufferAvailable.

/**
 * Tests that failures are propagated correctly if {@link
 * RemoteInputChannel#notifyBufferAvailable(int)} throws an exception. Also tests that a second
 * listener will be notified in this case.
 */
@Test
public void testFailureInNotifyBufferAvailable() throws Exception {
    // Setup
    final int numExclusiveBuffers = 1;
    final int numFloatingBuffers = 1;
    final int numTotalBuffers = numExclusiveBuffers + numFloatingBuffers;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(numTotalBuffers, 32);
    final SingleInputGate inputGate = createSingleInputGate(1);
    final RemoteInputChannel successfulRemoteIC = createRemoteInputChannel(inputGate);
    successfulRemoteIC.requestSubpartition();
    // late creation -> no exclusive buffers, also no requested subpartition in
    // successfulRemoteIC
    // (to trigger a failure in RemoteInputChannel#notifyBufferAvailable())
    final RemoteInputChannel failingRemoteIC = createRemoteInputChannel(inputGate);
    Buffer buffer = null;
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers);
        inputGate.setBufferPool(bufferPool);
        buffer = checkNotNull(bufferPool.requestBuffer());
        // trigger subscription to buffer pool
        failingRemoteIC.onSenderBacklog(1);
        successfulRemoteIC.onSenderBacklog(numExclusiveBuffers + 1);
        // recycling will call RemoteInputChannel#notifyBufferAvailable() which will fail and
        // this exception will be swallowed and set as an error in failingRemoteIC
        buffer.recycleBuffer();
        buffer = null;
        try {
            failingRemoteIC.checkError();
            fail("The input channel should have an error based on the failure in RemoteInputChannel#notifyBufferAvailable()");
        } catch (IOException e) {
            assertThat(e, hasProperty("cause", isA(IllegalStateException.class)));
        }
        // currently, the buffer is still enqueued in the bufferQueue of failingRemoteIC
        assertEquals(0, bufferPool.getNumberOfAvailableMemorySegments());
        buffer = successfulRemoteIC.requestBuffer();
        assertNull("buffer should still remain in failingRemoteIC", buffer);
        // releasing resources in failingRemoteIC should free the buffer again and immediately
        // recycle it into successfulRemoteIC
        failingRemoteIC.releaseAllResources();
        assertEquals(0, bufferPool.getNumberOfAvailableMemorySegments());
        buffer = successfulRemoteIC.requestBuffer();
        assertNotNull("no buffer given to successfulRemoteIC", buffer);
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, null, buffer, thrown, failingRemoteIC, successfulRemoteIC);
    }
}

Example 38 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method buildInputGate.

private SingleInputGate buildInputGate() throws IOException {
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(4, 4096);
    SingleInputGate inputGate = new SingleInputGateBuilder().setChannelFactory(InputChannelBuilder::buildRemoteChannel).setBufferPoolFactory(networkBufferPool.createBufferPool(1, 4)).setSegmentProvider(networkBufferPool).build();
    inputGate.setup();
    inputGate.requestPartitions();
    return inputGate;
}

Example 39 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method testAvailableBuffersEqualToRequiredBuffers.

/**
 * Tests to verify the behaviours of recycling floating and exclusive buffers if the number of
 * available buffers equals to required buffers.
 */
@Test
public void testAvailableBuffersEqualToRequiredBuffers() throws Exception {
    // Setup
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(16, 32);
    final int numFloatingBuffers = 14;
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate);
    inputGate.setInputChannels(inputChannel);
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = spy(networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers));
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        // Prepare the exclusive and floating buffers to verify recycle logic later
        final Buffer exclusiveBuffer = inputChannel.requestBuffer();
        assertNotNull(exclusiveBuffer);
        final Buffer floatingBuffer = bufferPool.requestBuffer();
        assertNotNull(floatingBuffer);
        verify(bufferPool, times(1)).requestBuffer();
        // Receive the producer's backlog
        inputChannel.onSenderBacklog(12);
        // The channel requests (backlog + numExclusiveBuffers) floating buffers from local pool
        // and gets enough floating buffers
        verify(bufferPool, times(14)).requestBuffer();
        verify(bufferPool, times(0)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 14 buffers available in the channel", 14, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 14 buffers required in the channel", 14, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 0 buffers available in local pool", 0, bufferPool.getNumberOfAvailableMemorySegments());
        // Recycle one floating buffer
        floatingBuffer.recycleBuffer();
        // The floating buffer is returned to local buffer directly because the channel is not
        // waiting
        // for floating buffers
        verify(bufferPool, times(14)).requestBuffer();
        verify(bufferPool, times(0)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 14 buffers available in the channel", 14, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 14 buffers required in the channel", 14, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 1 buffer available in local pool", 1, bufferPool.getNumberOfAvailableMemorySegments());
        // Recycle one exclusive buffer
        exclusiveBuffer.recycleBuffer();
        // Return one extra floating buffer to the local pool because the number of available
        // buffers
        // already equals to required buffers
        verify(bufferPool, times(14)).requestBuffer();
        verify(bufferPool, times(0)).addBufferListener(inputChannel.getBufferManager());
        assertEquals("There should be 14 buffers available in the channel", 14, inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be 14 buffers required in the channel", 14, inputChannel.getNumberOfRequiredBuffers());
        assertEquals("There should be 2 buffers available in local pool", 2, bufferPool.getNumberOfAvailableMemorySegments());
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, null, null, thrown, inputChannel);
    }
}

Example 40 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method testConcurrentOnSenderBacklogAndRecycle.

/**
 * Tests to verify that there is no race condition with two things running in parallel:
 * requesting floating buffers on sender backlog and some other thread recycling floating or
 * exclusive buffers.
 */
@Test
public void testConcurrentOnSenderBacklogAndRecycle() throws Exception {
    // Setup
    final int numExclusiveSegments = 120;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(248, 32);
    final int numFloatingBuffers = 128;
    final int backlog = 128;
    final ExecutorService executor = Executors.newFixedThreadPool(3);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = InputChannelTestUtils.createRemoteInputChannel(inputGate, numExclusiveSegments);
    inputGate.setInputChannels(inputChannel);
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        final Callable<Void> requestBufferTask = new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                for (int j = 1; j <= backlog; j++) {
                    inputChannel.onSenderBacklog(j);
                }
                return null;
            }
        };
        // Submit tasks and wait to finish
        submitTasksAndWaitForResults(executor, new Callable[] { recycleBufferTask(inputChannel, bufferPool, numExclusiveSegments, numFloatingBuffers), requestBufferTask });
        assertEquals("There should be " + inputChannel.getNumberOfRequiredBuffers() + " buffers available in channel.", inputChannel.getNumberOfRequiredBuffers(), inputChannel.getNumberOfAvailableBuffers());
        assertEquals("There should be no buffers available in local pool.", 0, bufferPool.getNumberOfAvailableMemorySegments());
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, executor, null, thrown, inputChannel);
    }
}