Examples with NetworkBufferPool org.apache.flink.runtime.io.network.buffer.NetworkBufferPool used on opensource projects

Example 56 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class DataBufferTest method createDataBuffer.

private DataBuffer createDataBuffer(int bufferPoolSize, int bufferSize, int numSubpartitions, int[] customReadOrder) throws IOException {
    NetworkBufferPool globalPool = new NetworkBufferPool(bufferPoolSize, bufferSize);
    BufferPool bufferPool = globalPool.createBufferPool(bufferPoolSize, bufferPoolSize);
    if (useHashBuffer) {
        return new HashBasedDataBuffer(bufferPool, numSubpartitions, bufferPoolSize, customReadOrder);
    } else {
        return new SortBasedDataBuffer(bufferPool, numSubpartitions, bufferSize, bufferPoolSize, customReadOrder);
    }
}

Example 57 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class NettyMessageClientDecoderDelegateTest method setup.

@Before
public void setup() throws IOException, InterruptedException {
    CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    networkBufferPool = new NetworkBufferPool(NUMBER_OF_BUFFER_RESPONSES, BUFFER_SIZE);
    channel = new EmbeddedChannel(new NettyMessageClientDecoderDelegate(handler));
    inputGate = createSingleInputGate(1, networkBufferPool);
    RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, new TestingPartitionRequestClient(), NUMBER_OF_BUFFER_RESPONSES);
    inputGate.setInputChannels(inputChannel);
    inputGate.setup();
    inputChannel.requestSubpartition();
    handler.addInputChannel(inputChannel);
    inputChannelId = inputChannel.getInputChannelId();
    SingleInputGate releasedInputGate = createSingleInputGate(1, networkBufferPool);
    RemoteInputChannel releasedInputChannel = new InputChannelBuilder().buildRemoteChannel(inputGate);
    releasedInputGate.close();
    handler.addInputChannel(releasedInputChannel);
    releasedInputChannelId = releasedInputChannel.getInputChannelId();
}

Example 58 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class NettyPartitionRequestClientTest method testResumeConsumption.

@Test
public void testResumeConsumption() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = createPartitionRequestClient(channel, handler, connectionReuseEnabled);
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
    try {
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        inputChannel.resumeConsumption();
        channel.runPendingTasks();
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(ResumeConsumption.class));
        assertEquals(inputChannel.getInputChannelId(), ((ResumeConsumption) readFromOutbound).receiverId);
        assertNull(channel.readOutbound());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}

Example 59 with NetworkBufferPool

use of org.apache.flink.runtime.io.network.buffer.NetworkBufferPool in project flink by apache.

the class NettyPartitionRequestClientTest method testPartitionRequestClientReuse.

@Test
public void testPartitionRequestClientReuse() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final NettyPartitionRequestClient client = createPartitionRequestClient(channel, handler, true);
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
    try {
        // Client should not be disposed in idle
        client.close(inputChannel);
        assertFalse(client.canBeDisposed());
        // Client should be disposed in error
        handler.notifyAllChannelsOfErrorAndClose(new RuntimeException());
        assertTrue(client.canBeDisposed());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}