Examples with NetworkBufferPool org.apache.flink.runtime.io.network.buffer.NetworkBufferPool used on opensource projects

Example 21 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method testGateNotifiedOnBarrierConversion.

@Test
public void testGateNotifiedOnBarrierConversion() throws IOException, InterruptedException {
    final int sequenceNumber = 0;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(1, 4096);
    try {
        SingleInputGate inputGate = new SingleInputGateBuilder().setBufferPoolFactory(networkBufferPool.createBufferPool(1, 1)).build();
        inputGate.setup();
        RemoteInputChannel channel = InputChannelBuilder.newBuilder().setConnectionManager(new TestVerifyConnectionManager(new TestVerifyPartitionRequestClient())).buildRemoteChannel(inputGate);
        channel.requestSubpartition();
        channel.onBuffer(toBuffer(new CheckpointBarrier(1L, 123L, alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), Integer.MAX_VALUE)), false), sequenceNumber, 0);
        // process announcement to allow the gate remember the SQN
        inputGate.pollNext();
        channel.convertToPriorityEvent(sequenceNumber);
        assertTrue(inputGate.getPriorityEventAvailableFuture().isDone());
    } finally {
        networkBufferPool.destroy();
    }
}

Example 22 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RemoteInputChannelTest method testFairDistributionFloatingBuffers.

/**
 * Tests to verify that the buffer pool will distribute available floating buffers among all the
 * channel listeners in a fair way.
 */
@Test
public void testFairDistributionFloatingBuffers() throws Exception {
    // Setup
    final int numExclusiveBuffers = 2;
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(12, 32);
    final int numFloatingBuffers = 3;
    final SingleInputGate inputGate = createSingleInputGate(3, networkBufferPool);
    final RemoteInputChannel[] inputChannels = new RemoteInputChannel[3];
    inputChannels[0] = createRemoteInputChannel(inputGate);
    inputChannels[1] = createRemoteInputChannel(inputGate);
    inputChannels[2] = createRemoteInputChannel(inputGate);
    inputGate.setInputChannels(inputChannels);
    Throwable thrown = null;
    try {
        final BufferPool bufferPool = spy(networkBufferPool.createBufferPool(numFloatingBuffers, numFloatingBuffers));
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputGate.requestPartitions();
        for (RemoteInputChannel inputChannel : inputChannels) {
            inputChannel.requestSubpartition();
        }
        // Exhaust all the floating buffers
        final List<Buffer> floatingBuffers = new ArrayList<>(numFloatingBuffers);
        for (int i = 0; i < numFloatingBuffers; i++) {
            Buffer buffer = bufferPool.requestBuffer();
            assertNotNull(buffer);
            floatingBuffers.add(buffer);
        }
        // and register as listeners as a result
        for (RemoteInputChannel inputChannel : inputChannels) {
            inputChannel.onSenderBacklog(8);
            verify(bufferPool, times(1)).addBufferListener(inputChannel.getBufferManager());
            assertEquals("There should be " + numExclusiveBuffers + " buffers available in the channel", numExclusiveBuffers, inputChannel.getNumberOfAvailableBuffers());
        }
        // Recycle three floating buffers to trigger notify buffer available
        for (Buffer buffer : floatingBuffers) {
            buffer.recycleBuffer();
        }
        for (RemoteInputChannel inputChannel : inputChannels) {
            assertEquals("There should be 3 buffers available in the channel", 3, inputChannel.getNumberOfAvailableBuffers());
            assertEquals("There should be 1 unannounced credits in the channel", 1, inputChannel.getUnannouncedCredit());
        }
    } catch (Throwable t) {
        thrown = t;
    } finally {
        cleanup(networkBufferPool, null, null, thrown, inputChannels);
    }
}

Example 23 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RecordWriterDelegateTest method setup.

@Before
public void setup() {
    assertEquals("Illegal memory segment size,", 0, memorySegmentSize % recordSize);
    globalPool = new NetworkBufferPool(numberOfBuffers, memorySegmentSize);
}

Example 24 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RecordWriterDelegateTest method createRecordWriter.

private RecordWriter createRecordWriter(NetworkBufferPool globalPool) throws Exception {
    final BufferPool localPool = globalPool.createBufferPool(1, 1, 1, Integer.MAX_VALUE);
    final ResultPartitionWriter partition = new ResultPartitionBuilder().setBufferPoolFactory(() -> localPool).build();
    partition.setup();
    return new RecordWriterBuilder().build(partition);
}

Example 25 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class RecordWriterTest method testIsAvailableOrNot.

/**
 * Tests that the RecordWriter is available iif the respective LocalBufferPool has at-least one
 * available buffer.
 */
@Test
public void testIsAvailableOrNot() throws Exception {
    // setup
    final NetworkBufferPool globalPool = new NetworkBufferPool(10, 128);
    final BufferPool localPool = globalPool.createBufferPool(1, 1, 1, Integer.MAX_VALUE);
    final ResultPartitionWriter resultPartition = new ResultPartitionBuilder().setBufferPoolFactory(() -> localPool).build();
    resultPartition.setup();
    final RecordWriter<?> recordWriter = createRecordWriter(resultPartition);
    try {
        // record writer is available because of initial available global pool
        assertTrue(recordWriter.getAvailableFuture().isDone());
        // request one buffer from the local pool to make it unavailable afterwards
        try (BufferBuilder bufferBuilder = localPool.requestBufferBuilder(0)) {
            assertNotNull(bufferBuilder);
            assertFalse(recordWriter.getAvailableFuture().isDone());
            // recycle the buffer to make the local pool available again
            final Buffer buffer = BufferBuilderTestUtils.buildSingleBuffer(bufferBuilder);
            buffer.recycleBuffer();
        }
        assertTrue(recordWriter.getAvailableFuture().isDone());
        assertEquals(recordWriter.AVAILABLE, recordWriter.getAvailableFuture());
    } finally {
        localPool.lazyDestroy();
        globalPool.destroy();
    }
}