Examples with SingleInputGate org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate used on opensource projects

Example 51 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testDoNotFailHandlerOnSingleChannelFailure.

@Test
public void testDoNotFailHandlerOnSingleChannelFailure() throws Exception {
    // Setup
    final int bufferSize = 1024;
    final String expectedMessage = "test exception on buffer";
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, bufferSize);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = new TestRemoteInputChannelForError(inputGate, expectedMessage);
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    try {
        inputGate.setInputChannels(inputChannel);
        inputGate.setup();
        inputGate.requestPartitions();
        handler.addInputChannel(inputChannel);
        final BufferResponse bufferResponse = createBufferResponse(TestBufferFactory.createBuffer(bufferSize), 0, inputChannel.getInputChannelId(), 1, new NetworkBufferAllocator(handler));
        // It will trigger an expected exception from TestRemoteInputChannelForError#onBuffer
        handler.channelRead(null, bufferResponse);
        // The handler should not be tagged as error for above excepted exception
        handler.checkError();
        try {
            // The input channel should be tagged as error and the respective exception is
            // thrown via #getNext
            inputGate.getNext();
        } catch (IOException ignored) {
            assertEquals(expectedMessage, ignored.getMessage());
        }
    } finally {
        // Cleanup
        releaseResource(inputGate, networkBufferPool);
    }
}

Example 52 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NettyShuffleEnvironmentTest method testRegisterTaskWithLimitedBuffers.

private void testRegisterTaskWithLimitedBuffers(int bufferPoolSize) throws Exception {
    final NettyShuffleEnvironment network = new NettyShuffleEnvironmentBuilder().setNumNetworkBuffers(bufferPoolSize).build();
    final ConnectionManager connManager = createDummyConnectionManager();
    int channels = 2;
    int rp4Channels = 4;
    int floatingBuffers = network.getConfiguration().floatingNetworkBuffersPerGate();
    int exclusiveBuffers = network.getConfiguration().networkBuffersPerChannel();
    int expectedBuffers = channels * exclusiveBuffers + floatingBuffers;
    int expectedRp4Buffers = rp4Channels * exclusiveBuffers + floatingBuffers;
    // result partitions
    ResultPartition rp1 = createPartition(network, ResultPartitionType.PIPELINED, channels);
    ResultPartition rp2 = createPartition(network, fileChannelManager, ResultPartitionType.BLOCKING, channels);
    ResultPartition rp3 = createPartition(network, ResultPartitionType.PIPELINED_BOUNDED, channels);
    ResultPartition rp4 = createPartition(network, ResultPartitionType.PIPELINED_BOUNDED, rp4Channels);
    final ResultPartition[] resultPartitions = new ResultPartition[] { rp1, rp2, rp3, rp4 };
    // input gates
    SingleInputGate ig1 = createSingleInputGate(network, ResultPartitionType.PIPELINED, channels);
    SingleInputGate ig2 = createSingleInputGate(network, ResultPartitionType.BLOCKING, channels);
    SingleInputGate ig3 = createSingleInputGate(network, ResultPartitionType.PIPELINED_BOUNDED, channels);
    SingleInputGate ig4 = createSingleInputGate(network, ResultPartitionType.PIPELINED_BOUNDED, rp4Channels);
    InputChannel[] ic1 = new InputChannel[channels];
    InputChannel[] ic2 = new InputChannel[channels];
    InputChannel[] ic3 = new InputChannel[channels];
    InputChannel[] ic4 = new InputChannel[rp4Channels];
    final SingleInputGate[] inputGates = new SingleInputGate[] { ig1, ig2, ig3, ig4 };
    ic4[0] = createRemoteInputChannel(ig4, 0, rp1, connManager);
    ic4[1] = createRemoteInputChannel(ig4, 0, rp2, connManager);
    ic4[2] = createRemoteInputChannel(ig4, 0, rp3, connManager);
    ic4[3] = createRemoteInputChannel(ig4, 0, rp4, connManager);
    ig4.setInputChannels(ic4);
    ic1[0] = createRemoteInputChannel(ig1, 1, rp1, connManager);
    ic1[1] = createRemoteInputChannel(ig1, 1, rp4, connManager);
    ig1.setInputChannels(ic1);
    ic2[0] = createRemoteInputChannel(ig2, 1, rp2, connManager);
    ic2[1] = createRemoteInputChannel(ig2, 2, rp4, connManager);
    ig2.setInputChannels(ic2);
    ic3[0] = createRemoteInputChannel(ig3, 1, rp3, connManager);
    ic3[1] = createRemoteInputChannel(ig3, 3, rp4, connManager);
    ig3.setInputChannels(ic3);
    Task.setupPartitionsAndGates(resultPartitions, inputGates);
    // verify buffer pools for the result partitions
    assertEquals(Integer.MAX_VALUE, rp1.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(Integer.MAX_VALUE, rp2.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(expectedBuffers, rp3.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(expectedRp4Buffers, rp4.getBufferPool().getMaxNumberOfMemorySegments());
    for (ResultPartition rp : resultPartitions) {
        assertEquals(rp.getNumberOfSubpartitions() + 1, rp.getBufferPool().getNumberOfRequiredMemorySegments());
        assertEquals(rp.getNumberOfSubpartitions() + 1, rp.getBufferPool().getNumBuffers());
    }
    // verify buffer pools for the input gates (NOTE: credit-based uses minimum required buffers
    // for exclusive buffers not managed by the buffer pool)
    assertEquals(1, ig1.getBufferPool().getNumberOfRequiredMemorySegments());
    assertEquals(1, ig2.getBufferPool().getNumberOfRequiredMemorySegments());
    assertEquals(1, ig3.getBufferPool().getNumberOfRequiredMemorySegments());
    assertEquals(1, ig4.getBufferPool().getNumberOfRequiredMemorySegments());
    assertEquals(floatingBuffers, ig1.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(floatingBuffers, ig2.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(floatingBuffers, ig3.getBufferPool().getMaxNumberOfMemorySegments());
    assertEquals(floatingBuffers, ig4.getBufferPool().getMaxNumberOfMemorySegments());
    verify(ig1, times(1)).setupChannels();
    verify(ig2, times(1)).setupChannels();
    verify(ig3, times(1)).setupChannels();
    verify(ig4, times(1)).setupChannels();
    for (ResultPartition rp : resultPartitions) {
        rp.release();
    }
    for (SingleInputGate ig : inputGates) {
        ig.close();
    }
    network.close();
}

Example 53 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NettyMessageClientDecoderDelegateTest method setup.

@Before
public void setup() throws IOException, InterruptedException {
    CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    networkBufferPool = new NetworkBufferPool(NUMBER_OF_BUFFER_RESPONSES, BUFFER_SIZE);
    channel = new EmbeddedChannel(new NettyMessageClientDecoderDelegate(handler));
    inputGate = createSingleInputGate(1, networkBufferPool);
    RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, new TestingPartitionRequestClient(), NUMBER_OF_BUFFER_RESPONSES);
    inputGate.setInputChannels(inputChannel);
    inputGate.setup();
    inputChannel.requestSubpartition();
    handler.addInputChannel(inputChannel);
    inputChannelId = inputChannel.getInputChannelId();
    SingleInputGate releasedInputGate = createSingleInputGate(1, networkBufferPool);
    RemoteInputChannel releasedInputChannel = new InputChannelBuilder().buildRemoteChannel(inputGate);
    releasedInputGate.close();
    handler.addInputChannel(releasedInputChannel);
    releasedInputChannelId = releasedInputChannel.getInputChannelId();
}

Example 54 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NettyPartitionRequestClientTest method testResumeConsumption.

@Test
public void testResumeConsumption() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = createPartitionRequestClient(channel, handler, connectionReuseEnabled);
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
    try {
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        inputChannel.resumeConsumption();
        channel.runPendingTasks();
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(ResumeConsumption.class));
        assertEquals(inputChannel.getInputChannelId(), ((ResumeConsumption) readFromOutbound).receiverId);
        assertNull(channel.readOutbound());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}

Example 55 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class NettyPartitionRequestClientTest method testPartitionRequestClientReuse.

@Test
public void testPartitionRequestClientReuse() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final NettyPartitionRequestClient client = createPartitionRequestClient(channel, handler, true);
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
    try {
        // Client should not be disposed in idle
        client.close(inputChannel);
        assertFalse(client.canBeDisposed());
        // Client should be disposed in error
        handler.notifyAllChannelsOfErrorAndClose(new RuntimeException());
        assertTrue(client.canBeDisposed());
    } finally {
        // Release all the buffer resources
        inputGate.close();
        networkBufferPool.destroyAllBufferPools();
        networkBufferPool.destroy();
    }
}