Examples with PartitionRequestClient org.apache.flink.runtime.io.network.PartitionRequestClient used on opensource projects

Example 1 with PartitionRequestClient

use of org.apache.flink.runtime.io.network.PartitionRequestClient in project flink by apache.

the class InputChannelTestUtils method createDummyConnectionManager.

public static ConnectionManager createDummyConnectionManager() throws Exception {
    final PartitionRequestClient mockClient = mock(PartitionRequestClient.class);
    final ConnectionManager connManager = mock(ConnectionManager.class);
    when(connManager.createPartitionRequestClient(any(ConnectionID.class))).thenReturn(mockClient);
    return connManager;
}

Example 2 with PartitionRequestClient

use of org.apache.flink.runtime.io.network.PartitionRequestClient in project flink by apache.

the class ClientTransportErrorHandlingTest method testExceptionOnWrite.

/**
 * Verifies that failed client requests via {@link PartitionRequestClient} are correctly
 * attributed to the respective {@link RemoteInputChannel}.
 */
@Test
public void testExceptionOnWrite() throws Exception {
    NettyProtocol protocol = new NettyProtocol(mock(ResultPartitionProvider.class), mock(TaskEventDispatcher.class)) {

        @Override
        public ChannelHandler[] getServerChannelHandlers() {
            return new ChannelHandler[0];
        }
    };
    // We need a real server and client in this test, because Netty's EmbeddedChannel is
    // not failing the ChannelPromise of failed writes.
    NettyServerAndClient serverAndClient = initServerAndClient(protocol, createConfig());
    Channel ch = connect(serverAndClient);
    NetworkClientHandler handler = getClientHandler(ch);
    // Last outbound handler throws Exception after 1st write
    ch.pipeline().addFirst(new ChannelOutboundHandlerAdapter() {

        int writeNum = 0;

        @Override
        public void write(ChannelHandlerContext ctx, Object msg, ChannelPromise promise) throws Exception {
            if (writeNum >= 1) {
                throw new RuntimeException("Expected test exception.");
            }
            writeNum++;
            ctx.write(msg, promise);
        }
    });
    PartitionRequestClient requestClient = new NettyPartitionRequestClient(ch, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));
    // Create input channels
    RemoteInputChannel[] rich = new RemoteInputChannel[] { createRemoteInputChannel(), createRemoteInputChannel() };
    final CountDownLatch sync = new CountDownLatch(1);
    // Do this with explicit synchronization. Otherwise this is not robust against slow timings
    // of the callback (e.g. we cannot just verify that it was called once, because there is
    // a chance that we do this too early).
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            sync.countDown();
            return null;
        }
    }).when(rich[1]).onError(isA(LocalTransportException.class));
    // First request is successful
    requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[0], 0);
    // Second request is *not* successful
    requestClient.requestSubpartition(new ResultPartitionID(), 0, rich[1], 0);
    // Wait for the notification and it could confirm all the request operations are done
    if (!sync.await(TestingUtils.TESTING_DURATION.toMillis(), TimeUnit.MILLISECONDS)) {
        fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION.toMillis() + " ms to be notified about the channel error.");
    }
    // Only the second channel should be notified about the error
    verify(rich[0], times(0)).onError(any(LocalTransportException.class));
    shutdown(serverAndClient);
}

Example 3 with PartitionRequestClient

use of org.apache.flink.runtime.io.network.PartitionRequestClient in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testAnnounceBufferSize.

@Test
public void testAnnounceBufferSize() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = new NettyPartitionRequestClient(channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(2, networkBufferPool);
    final RemoteInputChannel[] inputChannels = new RemoteInputChannel[2];
    inputChannels[0] = createRemoteInputChannel(inputGate, client);
    inputChannels[1] = createRemoteInputChannel(inputGate, client);
    try {
        inputGate.setInputChannels(inputChannels);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannels[0].requestSubpartition();
        inputChannels[1].requestSubpartition();
        channel.readOutbound();
        channel.readOutbound();
        inputGate.announceBufferSize(333);
        channel.runPendingTasks();
        NettyMessage.NewBufferSize readOutbound = channel.readOutbound();
        assertThat(readOutbound, instanceOf(NettyMessage.NewBufferSize.class));
        assertThat(readOutbound.receiverId, is(inputChannels[0].getInputChannelId()));
        assertThat(readOutbound.bufferSize, is(333));
        readOutbound = channel.readOutbound();
        assertThat(readOutbound.receiverId, is(inputChannels[1].getInputChannelId()));
        assertThat(readOutbound.bufferSize, is(333));
    } finally {
        releaseResource(inputGate, networkBufferPool);
        channel.close();
    }
}

Example 4 with PartitionRequestClient

use of org.apache.flink.runtime.io.network.PartitionRequestClient in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testNotifyCreditAvailable.

/**
 * Verifies that {@link RemoteInputChannel} is enqueued in the pipeline for notifying credits,
 * and verifies the behaviour of credit notification by triggering channel's writability
 * changed.
 */
@Test
public void testNotifyCreditAvailable() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final NetworkBufferAllocator allocator = new NetworkBufferAllocator(handler);
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = new NettyPartitionRequestClient(channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(2, networkBufferPool);
    final RemoteInputChannel[] inputChannels = new RemoteInputChannel[2];
    inputChannels[0] = createRemoteInputChannel(inputGate, client);
    inputChannels[1] = createRemoteInputChannel(inputGate, client);
    try {
        inputGate.setInputChannels(inputChannels);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannels[0].requestSubpartition();
        inputChannels[1].requestSubpartition();
        // The two input channels should send partition requests
        assertTrue(channel.isWritable());
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannels[0].getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(2, ((PartitionRequest) readFromOutbound).credit);
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(inputChannels[1].getInputChannelId(), ((PartitionRequest) readFromOutbound).receiverId);
        assertEquals(2, ((PartitionRequest) readFromOutbound).credit);
        // The buffer response will take one available buffer from input channel, and it will
        // trigger
        // requesting (backlog + numExclusiveBuffers - numAvailableBuffers) floating buffers
        final BufferResponse bufferResponse1 = createBufferResponse(TestBufferFactory.createBuffer(32), 0, inputChannels[0].getInputChannelId(), 1, allocator);
        final BufferResponse bufferResponse2 = createBufferResponse(TestBufferFactory.createBuffer(32), 0, inputChannels[1].getInputChannelId(), 1, allocator);
        handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse1);
        handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse2);
        assertEquals(2, inputChannels[0].getUnannouncedCredit());
        assertEquals(2, inputChannels[1].getUnannouncedCredit());
        channel.runPendingTasks();
        // The two input channels should notify credits availability via the writable channel
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(AddCredit.class));
        assertEquals(inputChannels[0].getInputChannelId(), ((AddCredit) readFromOutbound).receiverId);
        assertEquals(2, ((AddCredit) readFromOutbound).credit);
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(AddCredit.class));
        assertEquals(inputChannels[1].getInputChannelId(), ((AddCredit) readFromOutbound).receiverId);
        assertEquals(2, ((AddCredit) readFromOutbound).credit);
        assertNull(channel.readOutbound());
        ByteBuf channelBlockingBuffer = blockChannel(channel);
        // Trigger notify credits availability via buffer response on the condition of an
        // un-writable channel
        final BufferResponse bufferResponse3 = createBufferResponse(TestBufferFactory.createBuffer(32), 1, inputChannels[0].getInputChannelId(), 1, allocator);
        handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse3);
        assertEquals(1, inputChannels[0].getUnannouncedCredit());
        assertEquals(0, inputChannels[1].getUnannouncedCredit());
        channel.runPendingTasks();
        // The input channel will not notify credits via un-writable channel
        assertFalse(channel.isWritable());
        assertNull(channel.readOutbound());
        // Flush the buffer to make the channel writable again
        channel.flush();
        assertSame(channelBlockingBuffer, channel.readOutbound());
        // The input channel should notify credits via channel's writability changed event
        assertTrue(channel.isWritable());
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(AddCredit.class));
        assertEquals(1, ((AddCredit) readFromOutbound).credit);
        assertEquals(0, inputChannels[0].getUnannouncedCredit());
        assertEquals(0, inputChannels[1].getUnannouncedCredit());
        // no more messages
        assertNull(channel.readOutbound());
    } finally {
        releaseResource(inputGate, networkBufferPool);
        channel.close();
    }
}

Example 5 with PartitionRequestClient

use of org.apache.flink.runtime.io.network.PartitionRequestClient in project flink by apache.

the class CreditBasedPartitionRequestClientHandlerTest method testNotifyCreditAvailableAfterReleased.

/**
 * Verifies that {@link RemoteInputChannel} is enqueued in the pipeline, but {@link AddCredit}
 * message is not sent actually when this input channel is released.
 */
@Test
public void testNotifyCreditAvailableAfterReleased() throws Exception {
    final CreditBasedPartitionRequestClientHandler handler = new CreditBasedPartitionRequestClientHandler();
    final EmbeddedChannel channel = new EmbeddedChannel(handler);
    final PartitionRequestClient client = new NettyPartitionRequestClient(channel, handler, mock(ConnectionID.class), mock(PartitionRequestClientFactory.class));
    final NetworkBufferPool networkBufferPool = new NetworkBufferPool(10, 32);
    final SingleInputGate inputGate = createSingleInputGate(1, networkBufferPool);
    final RemoteInputChannel inputChannel = createRemoteInputChannel(inputGate, client);
    try {
        inputGate.setInputChannels(inputChannel);
        final BufferPool bufferPool = networkBufferPool.createBufferPool(6, 6);
        inputGate.setBufferPool(bufferPool);
        inputGate.setupChannels();
        inputChannel.requestSubpartition();
        // This should send the partition request
        Object readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(PartitionRequest.class));
        assertEquals(2, ((PartitionRequest) readFromOutbound).credit);
        // Trigger request floating buffers via buffer response to notify credits available
        final BufferResponse bufferResponse = createBufferResponse(TestBufferFactory.createBuffer(32), 0, inputChannel.getInputChannelId(), 1, new NetworkBufferAllocator(handler));
        handler.channelRead(mock(ChannelHandlerContext.class), bufferResponse);
        assertEquals(2, inputChannel.getUnannouncedCredit());
        // Release the input channel
        inputGate.close();
        // it should send a close request after releasing the input channel,
        // but will not notify credits for a released input channel.
        readFromOutbound = channel.readOutbound();
        assertThat(readFromOutbound, instanceOf(CloseRequest.class));
        channel.runPendingTasks();
        assertNull(channel.readOutbound());
    } finally {
        releaseResource(inputGate, networkBufferPool);
        channel.close();
    }
}