Examples with ResultPartitionProvider org.apache.flink.runtime.io.network.partition.ResultPartitionProvider used on opensource projects

Example 1 with ResultPartitionProvider

use of org.apache.flink.runtime.io.network.partition.ResultPartitionProvider in project flink by apache.

the class PartitionRequestQueueTest method testNotifyNewBufferSize.

@Test
public void testNotifyNewBufferSize() throws Exception {
    // given: Result partition and the reader for subpartition 0.
    ResultPartition parent = createResultPartition();
    BufferAvailabilityListener bufferAvailabilityListener = new NoOpBufferAvailablityListener();
    ResultSubpartitionView view = parent.createSubpartitionView(0, bufferAvailabilityListener);
    ResultPartitionProvider partitionProvider = (partitionId, index, availabilityListener) -> view;
    InputChannelID receiverId = new InputChannelID();
    PartitionRequestQueue queue = new PartitionRequestQueue();
    CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
    EmbeddedChannel channel = new EmbeddedChannel(queue);
    reader.requestSubpartitionView(partitionProvider, new ResultPartitionID(), 0);
    queue.notifyReaderCreated(reader);
    // when: New buffer size received.
    queue.notifyNewBufferSize(receiverId, 65);
    // and: New records emit.
    parent.emitRecord(ByteBuffer.allocate(128), 0);
    parent.emitRecord(ByteBuffer.allocate(10), 0);
    parent.emitRecord(ByteBuffer.allocate(60), 0);
    reader.notifyDataAvailable();
    channel.runPendingTasks();
    // then: Buffers of received size will be in outbound channel.
    Object data1 = channel.readOutbound();
    // The size can not be less than the first record in buffer.
    assertEquals(128, ((NettyMessage.BufferResponse) data1).buffer.getSize());
    Object data2 = channel.readOutbound();
    // The size should shrink up to notified buffer size.
    assertEquals(65, ((NettyMessage.BufferResponse) data2).buffer.getSize());
}

Example 2 with ResultPartitionProvider

use of org.apache.flink.runtime.io.network.partition.ResultPartitionProvider in project flink by apache.

the class PartitionRequestQueueTest method testEnqueueReaderByNotifyingBufferAndCredit.

/**
 * Tests {@link PartitionRequestQueue#enqueueAvailableReader(NetworkSequenceViewReader)},
 * verifying the reader would be enqueued in the pipeline iff it has both available credits and
 * buffers.
 */
@Test
public void testEnqueueReaderByNotifyingBufferAndCredit() throws Exception {
    // setup
    final ResultSubpartitionView view = new DefaultBufferResultSubpartitionView(10);
    ResultPartitionProvider partitionProvider = (partitionId, index, availabilityListener) -> view;
    final InputChannelID receiverId = new InputChannelID();
    final PartitionRequestQueue queue = new PartitionRequestQueue();
    final CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
    final EmbeddedChannel channel = new EmbeddedChannel(queue);
    reader.addCredit(-2);
    reader.requestSubpartitionView(partitionProvider, new ResultPartitionID(), 0);
    queue.notifyReaderCreated(reader);
    // block the channel so that we see an intermediate state in the test
    ByteBuf channelBlockingBuffer = blockChannel(channel);
    assertNull(channel.readOutbound());
    // Notify available buffers to trigger enqueue the reader
    final int notifyNumBuffers = 5;
    for (int i = 0; i < notifyNumBuffers; i++) {
        reader.notifyDataAvailable();
    }
    channel.runPendingTasks();
    // the reader is not enqueued in the pipeline because no credits are available
    // -> it should still have the same number of pending buffers
    assertEquals(0, queue.getAvailableReaders().size());
    assertTrue(reader.hasBuffersAvailable().isAvailable());
    assertFalse(reader.isRegisteredAsAvailable());
    assertEquals(0, reader.getNumCreditsAvailable());
    // Notify available credits to trigger enqueue the reader again
    final int notifyNumCredits = 3;
    for (int i = 1; i <= notifyNumCredits; i++) {
        queue.addCreditOrResumeConsumption(receiverId, viewReader -> viewReader.addCredit(1));
        // the reader is enqueued in the pipeline because it has both available buffers and
        // credits
        // since the channel is blocked though, we will not process anything and only enqueue
        // the
        // reader once
        assertTrue(reader.isRegisteredAsAvailable());
        // contains only (this) one!
        assertThat(queue.getAvailableReaders(), contains(reader));
        assertEquals(i, reader.getNumCreditsAvailable());
        assertTrue(reader.hasBuffersAvailable().isAvailable());
    }
    // Flush the buffer to make the channel writable again and see the final results
    channel.flush();
    assertSame(channelBlockingBuffer, channel.readOutbound());
    assertEquals(0, queue.getAvailableReaders().size());
    assertEquals(0, reader.getNumCreditsAvailable());
    assertTrue(reader.hasBuffersAvailable().isAvailable());
    assertFalse(reader.isRegisteredAsAvailable());
    for (int i = 1; i <= notifyNumCredits; i++) {
        assertThat(channel.readOutbound(), instanceOf(NettyMessage.BufferResponse.class));
    }
    assertNull(channel.readOutbound());
}

Example 3 with ResultPartitionProvider

use of org.apache.flink.runtime.io.network.partition.ResultPartitionProvider in project flink by apache.

the class PartitionRequestQueueTest method testProducerFailedException.

@Test
public void testProducerFailedException() throws Exception {
    PartitionRequestQueue queue = new PartitionRequestQueue();
    ResultPartitionProvider partitionProvider = mock(ResultPartitionProvider.class);
    ResultPartitionID rpid = new ResultPartitionID();
    BufferProvider bufferProvider = mock(BufferProvider.class);
    ResultSubpartitionView view = mock(ResultSubpartitionView.class);
    when(view.isReleased()).thenReturn(true);
    when(view.getFailureCause()).thenReturn(new RuntimeException("Expected test exception"));
    when(partitionProvider.createSubpartitionView(eq(rpid), eq(0), eq(bufferProvider), any(BufferAvailabilityListener.class))).thenReturn(view);
    EmbeddedChannel ch = new EmbeddedChannel(queue);
    SequenceNumberingViewReader seqView = new SequenceNumberingViewReader(new InputChannelID(), queue);
    seqView.requestSubpartitionView(partitionProvider, rpid, 0, bufferProvider);
    // Enqueue the erroneous view
    queue.notifyReaderNonEmpty(seqView);
    ch.runPendingTasks();
    // Read the enqueued msg
    Object msg = ch.readOutbound();
    assertEquals(msg.getClass(), NettyMessage.ErrorResponse.class);
    NettyMessage.ErrorResponse err = (NettyMessage.ErrorResponse) msg;
    assertTrue(err.cause instanceof CancelTaskException);
}

Example 4 with ResultPartitionProvider

use of org.apache.flink.runtime.io.network.partition.ResultPartitionProvider in project flink by apache.

the class PartitionRequestQueueTest method testEnqueueReaderByNotifyingEventBuffer.

/**
 * Tests {@link PartitionRequestQueue#enqueueAvailableReader(NetworkSequenceViewReader)},
 * verifying the reader would be enqueued in the pipeline if the next sending buffer is an event
 * even though it has no available credits.
 */
@Test
public void testEnqueueReaderByNotifyingEventBuffer() throws Exception {
    // setup
    final ResultSubpartitionView view = new NextIsEventResultSubpartitionView();
    ResultPartitionProvider partitionProvider = (partitionId, index, availabilityListener) -> view;
    final InputChannelID receiverId = new InputChannelID();
    final PartitionRequestQueue queue = new PartitionRequestQueue();
    final CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, 0, queue);
    final EmbeddedChannel channel = new EmbeddedChannel(queue);
    reader.requestSubpartitionView(partitionProvider, new ResultPartitionID(), 0);
    // block the channel so that we see an intermediate state in the test
    ByteBuf channelBlockingBuffer = blockChannel(channel);
    assertNull(channel.readOutbound());
    // Notify an available event buffer to trigger enqueue the reader
    reader.notifyDataAvailable();
    channel.runPendingTasks();
    // The reader is enqueued in the pipeline because the next buffer is an event, even though
    // no credits are available
    // contains only (this) one!
    assertThat(queue.getAvailableReaders(), contains(reader));
    assertEquals(0, reader.getNumCreditsAvailable());
    // Flush the buffer to make the channel writable again and see the final results
    channel.flush();
    assertSame(channelBlockingBuffer, channel.readOutbound());
    assertEquals(0, queue.getAvailableReaders().size());
    assertEquals(0, reader.getNumCreditsAvailable());
    assertNull(channel.readOutbound());
}

Example 5 with ResultPartitionProvider

use of org.apache.flink.runtime.io.network.partition.ResultPartitionProvider in project flink by apache.

the class PartitionRequestQueueTest method testBufferWriting.

private void testBufferWriting(ResultSubpartitionView view) throws IOException {
    // setup
    ResultPartitionProvider partitionProvider = (partitionId, index, availabilityListener) -> view;
    final InputChannelID receiverId = new InputChannelID();
    final PartitionRequestQueue queue = new PartitionRequestQueue();
    final CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, Integer.MAX_VALUE, queue);
    final EmbeddedChannel channel = new EmbeddedChannel(queue);
    reader.requestSubpartitionView(partitionProvider, new ResultPartitionID(), 0);
    // notify about buffer availability and encode one buffer
    reader.notifyDataAvailable();
    channel.runPendingTasks();
    Object read = channel.readOutbound();
    assertNotNull(read);
    if (read instanceof NettyMessage.ErrorResponse) {
        ((NettyMessage.ErrorResponse) read).cause.printStackTrace();
    }
    assertThat(read, instanceOf(NettyMessage.BufferResponse.class));
    read = channel.readOutbound();
    assertNull(read);
}