Examples with RemoteInputChannel org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel used on opensource projects

Example 11 with RemoteInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel in project flink by apache.

the class ClientTransportErrorHandlingTest method testExceptionOnRemoteClose.

/**
	 * Verifies that unexpected remote closes are reported as an instance of
	 * {@link RemoteTransportException}.
	 */
@Test
public void testExceptionOnRemoteClose() throws Exception {
    NettyProtocol protocol = new NettyProtocol() {

        @Override
        public ChannelHandler[] getServerChannelHandlers() {
            return new ChannelHandler[] { // Close on read
            new ChannelInboundHandlerAdapter() {

                @Override
                public void channelRead(ChannelHandlerContext ctx, Object msg) throws Exception {
                    ctx.channel().close();
                }
            } };
        }

        @Override
        public ChannelHandler[] getClientChannelHandlers() {
            return new PartitionRequestProtocol(mock(ResultPartitionProvider.class), mock(TaskEventDispatcher.class), mock(NetworkBufferPool.class)).getClientChannelHandlers();
        }
    };
    NettyServerAndClient serverAndClient = initServerAndClient(protocol, createConfig());
    Channel ch = connect(serverAndClient);
    PartitionRequestClientHandler handler = getClientHandler(ch);
    // Create input channels
    RemoteInputChannel[] rich = new RemoteInputChannel[] { createRemoteInputChannel(), createRemoteInputChannel() };
    final CountDownLatch sync = new CountDownLatch(rich.length);
    Answer<Void> countDownLatch = new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            sync.countDown();
            return null;
        }
    };
    for (RemoteInputChannel r : rich) {
        doAnswer(countDownLatch).when(r).onError(any(Throwable.class));
        handler.addInputChannel(r);
    }
    // Write something to trigger close by server
    ch.writeAndFlush(Unpooled.buffer().writerIndex(16));
    // Wait for the notification
    if (!sync.await(TestingUtils.TESTING_DURATION().toMillis(), TimeUnit.MILLISECONDS)) {
        fail("Timed out after waiting for " + TestingUtils.TESTING_DURATION().toMillis() + " ms to be notified about remote connection close.");
    }
    // All the registered channels should be notified.
    for (RemoteInputChannel r : rich) {
        verify(r).onError(isA(RemoteTransportException.class));
    }
    shutdown(serverAndClient);
}

Example 12 with RemoteInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel in project flink by apache.

the class ClientTransportErrorHandlingTest method testExceptionCaught.

/**
	 * Verifies that fired Exceptions are handled correctly by the pipeline.
	 */
@Test
public void testExceptionCaught() throws Exception {
    EmbeddedChannel ch = createEmbeddedChannel();
    PartitionRequestClientHandler handler = getClientHandler(ch);
    // Create input channels
    RemoteInputChannel[] rich = new RemoteInputChannel[] { createRemoteInputChannel(), createRemoteInputChannel() };
    for (RemoteInputChannel r : rich) {
        when(r.getInputChannelId()).thenReturn(new InputChannelID());
        handler.addInputChannel(r);
    }
    ch.pipeline().fireExceptionCaught(new Exception());
    try {
        // Exception should not reach end of pipeline...
        ch.checkException();
    } catch (Exception e) {
        fail("The exception reached the end of the pipeline and " + "was not handled correctly by the last handler.");
    }
    // ...but all the registered channels should be notified.
    for (RemoteInputChannel r : rich) {
        verify(r).onError(isA(LocalTransportException.class));
    }
}

Example 13 with RemoteInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel in project flink by apache.

the class ClientTransportErrorHandlingTest method testConnectionResetByPeer.

/**
	 * Verifies that "Connection reset by peer" Exceptions are special-cased and are reported as
	 * an instance of {@link RemoteTransportException}.
	 */
@Test
public void testConnectionResetByPeer() throws Throwable {
    EmbeddedChannel ch = createEmbeddedChannel();
    PartitionRequestClientHandler handler = getClientHandler(ch);
    RemoteInputChannel rich = addInputChannel(handler);
    final Throwable[] error = new Throwable[1];
    // Verify the Exception
    doAnswer(new Answer<Void>() {

        @Override
        public Void answer(InvocationOnMock invocation) throws Throwable {
            Throwable cause = (Throwable) invocation.getArguments()[0];
            try {
                assertEquals(RemoteTransportException.class, cause.getClass());
                assertNotEquals("Connection reset by peer", cause.getMessage());
                assertEquals(IOException.class, cause.getCause().getClass());
                assertEquals("Connection reset by peer", cause.getCause().getMessage());
            } catch (Throwable t) {
                error[0] = t;
            }
            return null;
        }
    }).when(rich).onError(any(Throwable.class));
    ch.pipeline().fireExceptionCaught(new IOException("Connection reset by peer"));
    assertNull(error[0]);
}

Example 14 with RemoteInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel in project flink by apache.

the class PartitionRequestClientHandlerTest method testAutoReadAfterUnsuccessfulStagedMessage.

/**
	 * Tests that an unsuccessful message decode call for a staged message
	 * does not leave the channel with auto read set to false.
	 */
@Test
@SuppressWarnings("unchecked")
public void testAutoReadAfterUnsuccessfulStagedMessage() throws Exception {
    PartitionRequestClientHandler handler = new PartitionRequestClientHandler();
    EmbeddedChannel channel = new EmbeddedChannel(handler);
    final AtomicReference<EventListener<Buffer>> listener = new AtomicReference<>();
    BufferProvider bufferProvider = mock(BufferProvider.class);
    when(bufferProvider.addListener(any(EventListener.class))).thenAnswer(new Answer<Boolean>() {

        @Override
        @SuppressWarnings("unchecked")
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            listener.set((EventListener<Buffer>) invocation.getArguments()[0]);
            return true;
        }
    });
    when(bufferProvider.requestBuffer()).thenReturn(null);
    InputChannelID channelId = new InputChannelID(0, 0);
    RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(channelId);
    // The 3rd staged msg has a null buffer provider
    when(inputChannel.getBufferProvider()).thenReturn(bufferProvider, bufferProvider, null);
    handler.addInputChannel(inputChannel);
    BufferResponse msg = createBufferResponse(createBuffer(true), 0, channelId);
    // Write 1st buffer msg. No buffer is available, therefore the buffer
    // should be staged and auto read should be set to false.
    assertTrue(channel.config().isAutoRead());
    channel.writeInbound(msg);
    // No buffer available, auto read false
    assertFalse(channel.config().isAutoRead());
    // Write more buffers... all staged.
    msg = createBufferResponse(createBuffer(true), 1, channelId);
    channel.writeInbound(msg);
    msg = createBufferResponse(createBuffer(true), 2, channelId);
    channel.writeInbound(msg);
    // Notify about buffer => handle 1st msg
    Buffer availableBuffer = createBuffer(false);
    listener.get().onEvent(availableBuffer);
    // Start processing of staged buffers (in run pending tasks). Make
    // sure that the buffer provider acts like it's destroyed.
    when(bufferProvider.addListener(any(EventListener.class))).thenReturn(false);
    when(bufferProvider.isDestroyed()).thenReturn(true);
    // Execute all tasks that are scheduled in the event loop. Further
    // eventLoop().execute() calls are directly executed, if they are
    // called in the scope of this call.
    channel.runPendingTasks();
    assertTrue(channel.config().isAutoRead());
}

Example 15 with RemoteInputChannel

use of org.apache.flink.runtime.io.network.partition.consumer.RemoteInputChannel in project flink by apache.

the class PartitionRequestClientHandlerTest method testReleaseInputChannelDuringDecode.

/**
	 * Tests a fix for FLINK-1627.
	 *
	 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
	 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
	 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
	 * provider of the receiving input channel, but the respective input channel is released (and
	 * the corresponding buffer provider destroyed), the handler did not notice this.
	 *
	 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
	 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
    // Mocks an input channel in a state as it was released during a decode.
    final BufferProvider bufferProvider = mock(BufferProvider.class);
    when(bufferProvider.requestBuffer()).thenReturn(null);
    when(bufferProvider.isDestroyed()).thenReturn(true);
    when(bufferProvider.addListener(any(EventListener.class))).thenReturn(false);
    final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
    when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);
    final BufferResponse ReceivedBuffer = createBufferResponse(TestBufferFactory.createBuffer(), 0, inputChannel.getInputChannelId());
    final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
    client.addInputChannel(inputChannel);
    client.channelRead(mock(ChannelHandlerContext.class), ReceivedBuffer);
}