Examples with InputChannelID org.apache.flink.runtime.io.network.partition.consumer.InputChannelID used on opensource projects

Example 6 with InputChannelID

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannelID in project flink by apache.

the class ClientTransportErrorHandlingTest method testExceptionCaught.

/**
	 * Verifies that fired Exceptions are handled correctly by the pipeline.
	 */
@Test
public void testExceptionCaught() throws Exception {
    EmbeddedChannel ch = createEmbeddedChannel();
    PartitionRequestClientHandler handler = getClientHandler(ch);
    // Create input channels
    RemoteInputChannel[] rich = new RemoteInputChannel[] { createRemoteInputChannel(), createRemoteInputChannel() };
    for (RemoteInputChannel r : rich) {
        when(r.getInputChannelId()).thenReturn(new InputChannelID());
        handler.addInputChannel(r);
    }
    ch.pipeline().fireExceptionCaught(new Exception());
    try {
        // Exception should not reach end of pipeline...
        ch.checkException();
    } catch (Exception e) {
        fail("The exception reached the end of the pipeline and " + "was not handled correctly by the last handler.");
    }
    // ...but all the registered channels should be notified.
    for (RemoteInputChannel r : rich) {
        verify(r).onError(isA(LocalTransportException.class));
    }
}

Example 7 with InputChannelID

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannelID in project flink by apache.

the class NettyMessageSerializationTest method testEncodeDecode.

@Test
public void testEncodeDecode() {
    {
        Buffer buffer = spy(new Buffer(MemorySegmentFactory.allocateUnpooledSegment(1024), mock(BufferRecycler.class)));
        ByteBuffer nioBuffer = buffer.getNioBuffer();
        for (int i = 0; i < 1024; i += 4) {
            nioBuffer.putInt(i);
        }
        NettyMessage.BufferResponse expected = new NettyMessage.BufferResponse(buffer, random.nextInt(), new InputChannelID());
        NettyMessage.BufferResponse actual = encodeAndDecode(expected);
        // Verify recycle has been called on buffer instance
        verify(buffer, times(1)).recycle();
        final ByteBuf retainedSlice = actual.getNettyBuffer();
        // Ensure not recycled and same size as original buffer
        assertEquals(1, retainedSlice.refCnt());
        assertEquals(1024, retainedSlice.readableBytes());
        nioBuffer = retainedSlice.nioBuffer();
        for (int i = 0; i < 1024; i += 4) {
            assertEquals(i, nioBuffer.getInt());
        }
        // Release the retained slice
        actual.releaseBuffer();
        assertEquals(0, retainedSlice.refCnt());
        assertEquals(expected.sequenceNumber, actual.sequenceNumber);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        {
            IllegalStateException expectedError = new IllegalStateException();
            InputChannelID receiverId = new InputChannelID();
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError, receiverId);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertEquals(receiverId, actual.receiverId);
        }
        {
            IllegalStateException expectedError = new IllegalStateException("Illegal illegal illegal");
            InputChannelID receiverId = new InputChannelID();
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError, receiverId);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertEquals(receiverId, actual.receiverId);
        }
        {
            IllegalStateException expectedError = new IllegalStateException("Illegal illegal illegal");
            NettyMessage.ErrorResponse expected = new NettyMessage.ErrorResponse(expectedError);
            NettyMessage.ErrorResponse actual = encodeAndDecode(expected);
            assertEquals(expected.cause.getClass(), actual.cause.getClass());
            assertEquals(expected.cause.getMessage(), actual.cause.getMessage());
            assertNull(actual.receiverId);
            assertTrue(actual.isFatalError());
        }
    }
    {
        NettyMessage.PartitionRequest expected = new NettyMessage.PartitionRequest(new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID()), random.nextInt(), new InputChannelID());
        NettyMessage.PartitionRequest actual = encodeAndDecode(expected);
        assertEquals(expected.partitionId, actual.partitionId);
        assertEquals(expected.queueIndex, actual.queueIndex);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.TaskEventRequest expected = new NettyMessage.TaskEventRequest(new IntegerTaskEvent(random.nextInt()), new ResultPartitionID(new IntermediateResultPartitionID(), new ExecutionAttemptID()), new InputChannelID());
        NettyMessage.TaskEventRequest actual = encodeAndDecode(expected);
        assertEquals(expected.event, actual.event);
        assertEquals(expected.partitionId, actual.partitionId);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.CancelPartitionRequest expected = new NettyMessage.CancelPartitionRequest(new InputChannelID());
        NettyMessage.CancelPartitionRequest actual = encodeAndDecode(expected);
        assertEquals(expected.receiverId, actual.receiverId);
    }
    {
        NettyMessage.CloseRequest expected = new NettyMessage.CloseRequest();
        NettyMessage.CloseRequest actual = encodeAndDecode(expected);
        assertEquals(expected.getClass(), actual.getClass());
    }
}

Example 8 with InputChannelID

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannelID in project flink by apache.

the class PartitionRequestClientHandlerTest method testAutoReadAfterUnsuccessfulStagedMessage.

/**
	 * Tests that an unsuccessful message decode call for a staged message
	 * does not leave the channel with auto read set to false.
	 */
@Test
@SuppressWarnings("unchecked")
public void testAutoReadAfterUnsuccessfulStagedMessage() throws Exception {
    PartitionRequestClientHandler handler = new PartitionRequestClientHandler();
    EmbeddedChannel channel = new EmbeddedChannel(handler);
    final AtomicReference<EventListener<Buffer>> listener = new AtomicReference<>();
    BufferProvider bufferProvider = mock(BufferProvider.class);
    when(bufferProvider.addListener(any(EventListener.class))).thenAnswer(new Answer<Boolean>() {

        @Override
        @SuppressWarnings("unchecked")
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            listener.set((EventListener<Buffer>) invocation.getArguments()[0]);
            return true;
        }
    });
    when(bufferProvider.requestBuffer()).thenReturn(null);
    InputChannelID channelId = new InputChannelID(0, 0);
    RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(channelId);
    // The 3rd staged msg has a null buffer provider
    when(inputChannel.getBufferProvider()).thenReturn(bufferProvider, bufferProvider, null);
    handler.addInputChannel(inputChannel);
    BufferResponse msg = createBufferResponse(createBuffer(true), 0, channelId);
    // Write 1st buffer msg. No buffer is available, therefore the buffer
    // should be staged and auto read should be set to false.
    assertTrue(channel.config().isAutoRead());
    channel.writeInbound(msg);
    // No buffer available, auto read false
    assertFalse(channel.config().isAutoRead());
    // Write more buffers... all staged.
    msg = createBufferResponse(createBuffer(true), 1, channelId);
    channel.writeInbound(msg);
    msg = createBufferResponse(createBuffer(true), 2, channelId);
    channel.writeInbound(msg);
    // Notify about buffer => handle 1st msg
    Buffer availableBuffer = createBuffer(false);
    listener.get().onEvent(availableBuffer);
    // Start processing of staged buffers (in run pending tasks). Make
    // sure that the buffer provider acts like it's destroyed.
    when(bufferProvider.addListener(any(EventListener.class))).thenReturn(false);
    when(bufferProvider.isDestroyed()).thenReturn(true);
    // Execute all tasks that are scheduled in the event loop. Further
    // eventLoop().execute() calls are directly executed, if they are
    // called in the scope of this call.
    channel.runPendingTasks();
    assertTrue(channel.config().isAutoRead());
}

Example 9 with InputChannelID

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannelID in project flink by apache.

the class PartitionRequestClientHandlerTest method testReleaseInputChannelDuringDecode.

/**
	 * Tests a fix for FLINK-1627.
	 *
	 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
	 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
	 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
	 * provider of the receiving input channel, but the respective input channel is released (and
	 * the corresponding buffer provider destroyed), the handler did not notice this.
	 *
	 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
	 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
    // Mocks an input channel in a state as it was released during a decode.
    final BufferProvider bufferProvider = mock(BufferProvider.class);
    when(bufferProvider.requestBuffer()).thenReturn(null);
    when(bufferProvider.isDestroyed()).thenReturn(true);
    when(bufferProvider.addListener(any(EventListener.class))).thenReturn(false);
    final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
    when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);
    final BufferResponse ReceivedBuffer = createBufferResponse(TestBufferFactory.createBuffer(), 0, inputChannel.getInputChannelId());
    final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
    client.addInputChannel(inputChannel);
    client.channelRead(mock(ChannelHandlerContext.class), ReceivedBuffer);
}

Example 10 with InputChannelID

use of org.apache.flink.runtime.io.network.partition.consumer.InputChannelID in project flink by apache.

the class PartitionRequestClientHandlerTest method testReceivePartitionNotFoundException.

/**
	 * Verifies that {@link RemoteInputChannel#onFailedPartitionRequest()} is called when a
	 * {@link PartitionNotFoundException} is received.
	 */
@Test
public void testReceivePartitionNotFoundException() throws Exception {
    // Minimal mock of a remote input channel
    final BufferProvider bufferProvider = mock(BufferProvider.class);
    when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer());
    final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
    when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);
    final ErrorResponse partitionNotFound = new ErrorResponse(new PartitionNotFoundException(new ResultPartitionID()), inputChannel.getInputChannelId());
    final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
    client.addInputChannel(inputChannel);
    // Mock channel context
    ChannelHandlerContext ctx = mock(ChannelHandlerContext.class);
    when(ctx.channel()).thenReturn(mock(Channel.class));
    client.channelActive(ctx);
    client.channelRead(ctx, partitionNotFound);
    verify(inputChannel, times(1)).onFailedPartitionRequest();
}