Examples with BufferResponse org.apache.flink.runtime.io.network.netty.NettyMessage.BufferResponse used on opensource projects

Example 1 with BufferResponse

use of org.apache.flink.runtime.io.network.netty.NettyMessage.BufferResponse in project flink by apache.

the class PartitionRequestClientHandlerTest method createBufferResponse.

/**
	 * Returns a deserialized buffer message as it would be received during runtime.
	 */
private BufferResponse createBufferResponse(Buffer buffer, int sequenceNumber, InputChannelID receivingChannelId) throws IOException {
    // Mock buffer to serialize
    BufferResponse resp = new BufferResponse(buffer, sequenceNumber, receivingChannelId);
    ByteBuf serialized = resp.write(UnpooledByteBufAllocator.DEFAULT);
    // Skip general header bytes
    serialized.readBytes(NettyMessage.HEADER_LENGTH);
    BufferResponse deserialized = new BufferResponse();
    // Deserialize the bytes again. We have to go this way, because we only partly deserialize
    // the header of the response and wait for a buffer from the buffer pool to copy the payload
    // data into.
    deserialized.readFrom(serialized);
    return deserialized;
}

Example 2 with BufferResponse

use of org.apache.flink.runtime.io.network.netty.NettyMessage.BufferResponse in project flink by apache.

the class PartitionRequestClientHandlerTest method testReceiveEmptyBuffer.

/**
	 * Tests a fix for FLINK-1761.
	 *
	 * <p> FLINK-1761 discovered an IndexOutOfBoundsException, when receiving buffers of size 0.
	 */
@Test
public void testReceiveEmptyBuffer() throws Exception {
    // Minimal mock of a remote input channel
    final BufferProvider bufferProvider = mock(BufferProvider.class);
    when(bufferProvider.requestBuffer()).thenReturn(TestBufferFactory.createBuffer());
    final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
    when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);
    // An empty buffer of size 0
    final Buffer emptyBuffer = TestBufferFactory.createBuffer();
    emptyBuffer.setSize(0);
    final BufferResponse receivedBuffer = createBufferResponse(emptyBuffer, 0, inputChannel.getInputChannelId());
    final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
    client.addInputChannel(inputChannel);
    // Read the empty buffer
    client.channelRead(mock(ChannelHandlerContext.class), receivedBuffer);
    // This should not throw an exception
    verify(inputChannel, never()).onError(any(Throwable.class));
}

Example 3 with BufferResponse

use of org.apache.flink.runtime.io.network.netty.NettyMessage.BufferResponse in project flink by apache.

the class PartitionRequestQueue method writeAndFlushNextMessageIfPossible.

private void writeAndFlushNextMessageIfPossible(final Channel channel) throws IOException {
    if (fatalError) {
        return;
    }
    // The logic here is very similar to the combined input gate and local
    // input channel logic. You can think of this class acting as the input
    // gate and the consumed views as the local input channels.
    BufferAndAvailability next = null;
    try {
        if (channel.isWritable()) {
            while (true) {
                SequenceNumberingViewReader reader = nonEmptyReader.poll();
                // of the write callbacks that are executed after each write.
                if (reader == null) {
                    return;
                }
                next = reader.getNextBuffer();
                if (next == null) {
                    if (reader.isReleased()) {
                        markAsReleased(reader.getReceiverId());
                        Throwable cause = reader.getFailureCause();
                        if (cause != null) {
                            ErrorResponse msg = new ErrorResponse(new ProducerFailedException(cause), reader.getReceiverId());
                            ctx.writeAndFlush(msg);
                        }
                    } else {
                        IllegalStateException err = new IllegalStateException("Bug in Netty consumer logic: reader queue got notified by partition " + "about available data, but none was available.");
                        handleException(ctx.channel(), err);
                        return;
                    }
                } else {
                    // "non-empty" notification will come for that reader from the queue.
                    if (next.moreAvailable()) {
                        nonEmptyReader.add(reader);
                    }
                    BufferResponse msg = new BufferResponse(next.buffer(), reader.getSequenceNumber(), reader.getReceiverId());
                    if (isEndOfPartitionEvent(next.buffer())) {
                        reader.notifySubpartitionConsumed();
                        reader.releaseAllResources();
                        markAsReleased(reader.getReceiverId());
                    }
                    // Write and flush and wait until this is done before
                    // trying to continue with the next buffer.
                    channel.writeAndFlush(msg).addListener(writeListener);
                    return;
                }
            }
        }
    } catch (Throwable t) {
        if (next != null) {
            next.buffer().recycle();
        }
        throw new IOException(t.getMessage(), t);
    }
}

Example 4 with BufferResponse

use of org.apache.flink.runtime.io.network.netty.NettyMessage.BufferResponse in project flink by apache.

the class PartitionRequestClientHandlerTest method testAutoReadAfterUnsuccessfulStagedMessage.

/**
	 * Tests that an unsuccessful message decode call for a staged message
	 * does not leave the channel with auto read set to false.
	 */
@Test
@SuppressWarnings("unchecked")
public void testAutoReadAfterUnsuccessfulStagedMessage() throws Exception {
    PartitionRequestClientHandler handler = new PartitionRequestClientHandler();
    EmbeddedChannel channel = new EmbeddedChannel(handler);
    final AtomicReference<EventListener<Buffer>> listener = new AtomicReference<>();
    BufferProvider bufferProvider = mock(BufferProvider.class);
    when(bufferProvider.addListener(any(EventListener.class))).thenAnswer(new Answer<Boolean>() {

        @Override
        @SuppressWarnings("unchecked")
        public Boolean answer(InvocationOnMock invocation) throws Throwable {
            listener.set((EventListener<Buffer>) invocation.getArguments()[0]);
            return true;
        }
    });
    when(bufferProvider.requestBuffer()).thenReturn(null);
    InputChannelID channelId = new InputChannelID(0, 0);
    RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(channelId);
    // The 3rd staged msg has a null buffer provider
    when(inputChannel.getBufferProvider()).thenReturn(bufferProvider, bufferProvider, null);
    handler.addInputChannel(inputChannel);
    BufferResponse msg = createBufferResponse(createBuffer(true), 0, channelId);
    // Write 1st buffer msg. No buffer is available, therefore the buffer
    // should be staged and auto read should be set to false.
    assertTrue(channel.config().isAutoRead());
    channel.writeInbound(msg);
    // No buffer available, auto read false
    assertFalse(channel.config().isAutoRead());
    // Write more buffers... all staged.
    msg = createBufferResponse(createBuffer(true), 1, channelId);
    channel.writeInbound(msg);
    msg = createBufferResponse(createBuffer(true), 2, channelId);
    channel.writeInbound(msg);
    // Notify about buffer => handle 1st msg
    Buffer availableBuffer = createBuffer(false);
    listener.get().onEvent(availableBuffer);
    // Start processing of staged buffers (in run pending tasks). Make
    // sure that the buffer provider acts like it's destroyed.
    when(bufferProvider.addListener(any(EventListener.class))).thenReturn(false);
    when(bufferProvider.isDestroyed()).thenReturn(true);
    // Execute all tasks that are scheduled in the event loop. Further
    // eventLoop().execute() calls are directly executed, if they are
    // called in the scope of this call.
    channel.runPendingTasks();
    assertTrue(channel.config().isAutoRead());
}

Example 5 with BufferResponse

use of org.apache.flink.runtime.io.network.netty.NettyMessage.BufferResponse in project flink by apache.

the class PartitionRequestClientHandlerTest method testReleaseInputChannelDuringDecode.

/**
	 * Tests a fix for FLINK-1627.
	 *
	 * <p> FLINK-1627 discovered a race condition, which could lead to an infinite loop when a
	 * receiver was cancelled during a certain time of decoding a message. The test reproduces the
	 * input, which lead to the infinite loop: when the handler gets a reference to the buffer
	 * provider of the receiving input channel, but the respective input channel is released (and
	 * the corresponding buffer provider destroyed), the handler did not notice this.
	 *
	 * @see <a href="https://issues.apache.org/jira/browse/FLINK-1627">FLINK-1627</a>
	 */
@Test(timeout = 60000)
@SuppressWarnings("unchecked")
public void testReleaseInputChannelDuringDecode() throws Exception {
    // Mocks an input channel in a state as it was released during a decode.
    final BufferProvider bufferProvider = mock(BufferProvider.class);
    when(bufferProvider.requestBuffer()).thenReturn(null);
    when(bufferProvider.isDestroyed()).thenReturn(true);
    when(bufferProvider.addListener(any(EventListener.class))).thenReturn(false);
    final RemoteInputChannel inputChannel = mock(RemoteInputChannel.class);
    when(inputChannel.getInputChannelId()).thenReturn(new InputChannelID());
    when(inputChannel.getBufferProvider()).thenReturn(bufferProvider);
    final BufferResponse ReceivedBuffer = createBufferResponse(TestBufferFactory.createBuffer(), 0, inputChannel.getInputChannelId());
    final PartitionRequestClientHandler client = new PartitionRequestClientHandler();
    client.addInputChannel(inputChannel);
    client.channelRead(mock(ChannelHandlerContext.class), ReceivedBuffer);
}