Examples with SingleInputGate org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate used on opensource projects

Example 11 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionRemoteChannels.

@Test
public void testFairConsumptionRemoteChannels() throws Exception {
    final int numberOfChannels = 37;
    final int buffersPerChannel = 27;
    final Buffer mockBuffer = TestBufferFactory.createBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
    final ConnectionManager connManager = createDummyConnectionManager();
    final RemoteInputChannel[] channels = new RemoteInputChannel[numberOfChannels];
    final int[] channelSequenceNums = new int[numberOfChannels];
    for (int i = 0; i < numberOfChannels; i++) {
        RemoteInputChannel channel = createRemoteInputChannel(gate, i, connManager);
        channels[i] = channel;
    }
    channels[11].onBuffer(mockBuffer, 0, -1);
    channelSequenceNums[11]++;
    setupInputGate(gate, channels);
    // read all the buffers and the EOF event
    for (int i = 0; i < numberOfChannels * buffersPerChannel; i++) {
        assertNotNull(gate.getNext());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (RemoteInputChannel channel : channels) {
            int size = channel.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == (min + 1));
        if (i % (2 * numberOfChannels) == 0) {
            // add three buffers to each channel, in random order
            fillRandom(channels, channelSequenceNums, 3, mockBuffer);
        }
    }
}

Example 12 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class InputGateFairnessTest method testFairConsumptionRemoteChannelsPreFilled.

@Test
public void testFairConsumptionRemoteChannelsPreFilled() throws Exception {
    final int numberOfChannels = 37;
    final int buffersPerChannel = 27;
    final Buffer mockBuffer = TestBufferFactory.createBuffer(42);
    // ----- create some source channels and fill them with buffers -----
    final SingleInputGate gate = createFairnessVerifyingInputGate(numberOfChannels);
    final ConnectionManager connManager = createDummyConnectionManager();
    final RemoteInputChannel[] channels = new RemoteInputChannel[numberOfChannels];
    for (int i = 0; i < numberOfChannels; i++) {
        RemoteInputChannel channel = createRemoteInputChannel(gate, i, connManager);
        channels[i] = channel;
        for (int p = 0; p < buffersPerChannel; p++) {
            channel.onBuffer(mockBuffer, p, -1);
        }
        channel.onBuffer(EventSerializer.toBuffer(EndOfPartitionEvent.INSTANCE, false), buffersPerChannel, -1);
    }
    gate.setInputChannels(channels);
    gate.setup();
    gate.requestPartitions();
    // read all the buffers and the EOF event
    for (int i = numberOfChannels * (buffersPerChannel + 1); i > 0; --i) {
        assertNotNull(gate.getNext());
        int min = Integer.MAX_VALUE;
        int max = 0;
        for (RemoteInputChannel channel : channels) {
            int size = channel.getNumberOfQueuedBuffers();
            min = Math.min(min, size);
            max = Math.max(max, size);
        }
        assertTrue(max == min || max == (min + 1));
    }
    assertFalse(gate.getNext().isPresent());
}

Example 13 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testReadBufferResponseWithReleasingOrRemovingChannel.

private void testReadBufferResponseWithReleasingOrRemovingChannel(boolean isRemoved, boolean readBeforeReleasingOrRemoving) throws Exception {
    int bufferSize = 1024;
    NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, bufferSize);
    SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    RemoteInputChannel inputChannel = new InputChannelBuilder().buildRemoteChannel(inputGate);
    inputGate.setInputChannels(inputChannel);
    inputGate.setup();
    CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    EmbeddedChannel embeddedChannel = new EmbeddedChannel(handler);
    handler.addInputChannel(inputChannel);
    try {
        if (!readBeforeReleasingOrRemoving) {
            // Release the channel.
            inputGate.close();
            if (isRemoved) {
                handler.removeInputChannel(inputChannel);
            }
        }
        BufferResponse bufferResponse = createBufferResponse(TestBufferFactory.createBuffer(bufferSize), 0, inputChannel.getInputChannelId(), 1, new NetworkBufferAllocator(handler));
        if (readBeforeReleasingOrRemoving) {
            // Release the channel.
            inputGate.close();
            if (isRemoved) {
                handler.removeInputChannel(inputChannel);
            }
        }
        handler.channelRead(null, bufferResponse);
        assertEquals(0, inputChannel.getNumberOfQueuedBuffers());
        if (!readBeforeReleasingOrRemoving) {
            assertNull(bufferResponse.getBuffer());
        } else {
            assertNotNull(bufferResponse.getBuffer());
            assertTrue(bufferResponse.getBuffer().isRecycled());
        }
        embeddedChannel.runScheduledPendingTasks();
        NettyMessage.CancelPartitionRequest cancelPartitionRequest = embeddedChannel.readOutbound();
        assertNotNull(cancelPartitionRequest);
        assertEquals(inputChannel.getInputChannelId(), cancelPartitionRequest.receiverId);
    } finally {
        releaseResource(inputGate, networkBufferPool);
        embeddedChannel.close();
    }
}

Example 14 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testThrowExceptionForNoAvailableBuffer.

/**
 * Verifies that {@link RemoteInputChannel#onError(Throwable)} is called when a {@link
 * BufferResponse} is received but no available buffer in input channel.
 */
@Test
public void testThrowExceptionForNoAvailableBuffer() throws Exception {
    final SingleInputGate inputGate = createSingleInputGate(1);
    final RemoteInputChannel inputChannel = spy(InputChannelBuilder.newBuilder().buildRemoteChannel(inputGate));
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    handler.addInputChannel(inputChannel);
    assertEquals("There should be no buffers available in the channel.", 0, inputChannel.getNumberOfAvailableBuffers());
    final BufferResponse bufferResponse = createBufferResponse(TestBufferFactory.createBuffer(TestBufferFactory.BUFFER_SIZE), 0, inputChannel.getInputChannelId(), 2, new NetworkBufferAllocator(handler));
    assertNull(bufferResponse.getBuffer());
    handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);
    verify(inputChannel, times(1)).onError(any(IllegalStateException.class));
}

Example 15 with SingleInputGate

use of org.apache.flink.runtime.io.network.partition.consumer.SingleInputGate in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testAnnounceBufferSize.

@Test
public void testAnnounceBufferSize() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = new NettyPartitionRequestClient(channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(2, networkBufferPool);
    final RemoteInputChannel[] inputChannels = new RemoteInputChannel[2];
    inputChannels[0] = createRemoteInputChannel(inputGate, client);
    inputChannels[1] = createRemoteInputChannel(inputGate, client);
    try {
        inputGate.setInputChannels(inputChannels);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannels[0].requestSubpartition();
        inputChannels[1].requestSubpartition();
        channel.readOutbound();
        channel.readOutbound();
        inputGate.announceBufferSize(333);
        channel.runPendingTasks();
        NettyMessage.NewBufferSize readOutbound = channel.readOutbound();
        assertThat(readOutbound, instanceOf(NettyMessage.NewBufferSize.class));
        assertThat(readOutbound.receiverId, is(inputChannels[0].getInputChannelId()));
        assertThat(readOutbound.bufferSize, is(333));
        readOutbound = channel.readOutbound();
        assertThat(readOutbound.receiverId, is(inputChannels[1].getInputChannelId()));
        assertThat(readOutbound.bufferSize, is(333));
    } finally {
        releaseResource(inputGate, networkBufferPool);
        channel.close();
    }
}