Search in sources :

Example 1 with PipelinedSubpartitionView

use of org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionView in project flink by apache.

the class PartitionRequestQueueTest method testEnqueueReaderByResumingConsumption.

/**
 * Tests {@link PartitionRequestQueue#enqueueAvailableReader(NetworkSequenceViewReader)},
 * verifying the reader would be enqueued in the pipeline after resuming data consumption if
 * there are credit and data available.
 */
@Test
public void testEnqueueReaderByResumingConsumption() throws Exception {
    PipelinedSubpartition subpartition = PipelinedSubpartitionTest.createPipelinedSubpartition();
    Buffer.DataType dataType1 = Buffer.DataType.ALIGNED_CHECKPOINT_BARRIER;
    Buffer.DataType dataType2 = Buffer.DataType.DATA_BUFFER;
    subpartition.add(createEventBufferConsumer(4096, dataType1));
    subpartition.add(createEventBufferConsumer(4096, dataType2));
    BufferAvailabilityListener bufferAvailabilityListener = new NoOpBufferAvailablityListener();
    PipelinedSubpartitionView view = subpartition.createReadView(bufferAvailabilityListener);
    ResultPartitionProvider partitionProvider = (partitionId, index, availabilityListener) -> view;
    InputChannelID receiverId = new InputChannelID();
    PartitionRequestQueue queue = new PartitionRequestQueue();
    CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, 2, queue);
    EmbeddedChannel channel = new EmbeddedChannel(queue);
    reader.requestSubpartitionView(partitionProvider, new ResultPartitionID(), 0);
    queue.notifyReaderCreated(reader);
    assertTrue(reader.getAvailabilityAndBacklog().isAvailable());
    reader.notifyDataAvailable();
    channel.runPendingTasks();
    assertFalse(reader.getAvailabilityAndBacklog().isAvailable());
    assertEquals(1, subpartition.unsynchronizedGetNumberOfQueuedBuffers());
    queue.addCreditOrResumeConsumption(receiverId, NetworkSequenceViewReader::resumeConsumption);
    assertFalse(reader.getAvailabilityAndBacklog().isAvailable());
    assertEquals(0, subpartition.unsynchronizedGetNumberOfQueuedBuffers());
    Object data1 = channel.readOutbound();
    assertEquals(dataType1, ((NettyMessage.BufferResponse) data1).buffer.getDataType());
    Object data2 = channel.readOutbound();
    assertEquals(dataType2, ((NettyMessage.BufferResponse) data2).buffer.getDataType());
}
Also used : ByteBuffer(java.nio.ByteBuffer) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog) TestBufferFactory(org.apache.flink.runtime.io.network.util.TestBufferFactory) BeforeClass(org.junit.BeforeClass) ResultPartitionType(org.apache.flink.runtime.io.network.partition.ResultPartitionType) ByteBuffer(java.nio.ByteBuffer) FileChannelManager(org.apache.flink.runtime.io.disk.FileChannelManager) NetworkSequenceViewReader(org.apache.flink.runtime.io.network.NetworkSequenceViewReader) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) Assert.assertSame(org.junit.Assert.assertSame) Assert.assertThat(org.junit.Assert.assertThat) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) BufferAvailabilityListener(org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener) BufferBuilderTestUtils.createEventBufferConsumer(org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createEventBufferConsumer) ClassRule(org.junit.ClassRule) Nullable(javax.annotation.Nullable) NoOpResultSubpartitionView(org.apache.flink.runtime.io.network.partition.NoOpResultSubpartitionView) AfterClass(org.junit.AfterClass) PipelinedSubpartitionView(org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionView) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) Assert.assertNotNull(org.junit.Assert.assertNotNull) Unpooled(org.apache.flink.shaded.netty4.io.netty.buffer.Unpooled) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) ByteBuf(org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) PipelinedSubpartition(org.apache.flink.runtime.io.network.partition.PipelinedSubpartition) Assert.assertNull(org.junit.Assert.assertNull) Matchers.contains(org.hamcrest.Matchers.contains) Assert.assertFalse(org.junit.Assert.assertFalse) NoOpFileChannelManager(org.apache.flink.runtime.io.disk.NoOpFileChannelManager) PipelinedSubpartitionTest(org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionTest) FileChannelManagerImpl(org.apache.flink.runtime.io.disk.FileChannelManagerImpl) ResultPartitionManager(org.apache.flink.runtime.io.network.partition.ResultPartitionManager) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) EmbeddedChannel(org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel) PartitionTestUtils.createPartition(org.apache.flink.runtime.io.network.partition.PartitionTestUtils.createPartition) ResultPartitionProvider(org.apache.flink.runtime.io.network.partition.ResultPartitionProvider) InputChannelID(org.apache.flink.runtime.io.network.partition.consumer.InputChannelID) TemporaryFolder(org.junit.rules.TemporaryFolder) Assert.assertEquals(org.junit.Assert.assertEquals) PipelinedSubpartition(org.apache.flink.runtime.io.network.partition.PipelinedSubpartition) EmbeddedChannel(org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel) PipelinedSubpartitionView(org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionView) InputChannelID(org.apache.flink.runtime.io.network.partition.consumer.InputChannelID) BufferAvailabilityListener(org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID) NetworkSequenceViewReader(org.apache.flink.runtime.io.network.NetworkSequenceViewReader) ResultPartitionProvider(org.apache.flink.runtime.io.network.partition.ResultPartitionProvider) Test(org.junit.Test) PipelinedSubpartitionTest(org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionTest)

Example 2 with PipelinedSubpartitionView

use of org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionView in project flink by apache.

the class PartitionRequestQueueTest method testAnnounceBacklog.

@Test
public void testAnnounceBacklog() throws Exception {
    PipelinedSubpartition subpartition = PipelinedSubpartitionTest.createPipelinedSubpartition();
    subpartition.add(createEventBufferConsumer(4096, Buffer.DataType.DATA_BUFFER));
    subpartition.add(createEventBufferConsumer(4096, Buffer.DataType.DATA_BUFFER));
    PipelinedSubpartitionView view = subpartition.createReadView(new NoOpBufferAvailablityListener());
    ResultPartitionProvider partitionProvider = (partitionId, index, availabilityListener) -> view;
    PartitionRequestQueue queue = new PartitionRequestQueue();
    InputChannelID receiverId = new InputChannelID();
    CreditBasedSequenceNumberingViewReader reader = new CreditBasedSequenceNumberingViewReader(receiverId, 0, queue);
    EmbeddedChannel channel = new EmbeddedChannel(queue);
    reader.requestSubpartitionView(partitionProvider, new ResultPartitionID(), 0);
    queue.notifyReaderCreated(reader);
    reader.notifyDataAvailable();
    channel.runPendingTasks();
    Object data = channel.readOutbound();
    assertTrue(data instanceof NettyMessage.BacklogAnnouncement);
    NettyMessage.BacklogAnnouncement announcement = (NettyMessage.BacklogAnnouncement) data;
    assertEquals(receiverId, announcement.receiverId);
    assertEquals(subpartition.getBuffersInBacklogUnsafe(), announcement.backlog);
    subpartition.release();
    reader.notifyDataAvailable();
    channel.runPendingTasks();
    assertNotNull(channel.readOutbound());
}
Also used : BufferAndBacklog(org.apache.flink.runtime.io.network.partition.ResultSubpartition.BufferAndBacklog) TestBufferFactory(org.apache.flink.runtime.io.network.util.TestBufferFactory) BeforeClass(org.junit.BeforeClass) ResultPartitionType(org.apache.flink.runtime.io.network.partition.ResultPartitionType) ByteBuffer(java.nio.ByteBuffer) FileChannelManager(org.apache.flink.runtime.io.disk.FileChannelManager) NetworkSequenceViewReader(org.apache.flink.runtime.io.network.NetworkSequenceViewReader) ResultPartition(org.apache.flink.runtime.io.network.partition.ResultPartition) Assert.assertSame(org.junit.Assert.assertSame) Assert.assertThat(org.junit.Assert.assertThat) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) BufferAvailabilityListener(org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener) BufferBuilderTestUtils.createEventBufferConsumer(org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createEventBufferConsumer) ClassRule(org.junit.ClassRule) Nullable(javax.annotation.Nullable) NoOpResultSubpartitionView(org.apache.flink.runtime.io.network.partition.NoOpResultSubpartitionView) AfterClass(org.junit.AfterClass) PipelinedSubpartitionView(org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionView) NettyShuffleEnvironmentBuilder(org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder) Assert.assertNotNull(org.junit.Assert.assertNotNull) Unpooled(org.apache.flink.shaded.netty4.io.netty.buffer.Unpooled) Assert.assertTrue(org.junit.Assert.assertTrue) Test(org.junit.Test) IOException(java.io.IOException) NettyShuffleEnvironment(org.apache.flink.runtime.io.network.NettyShuffleEnvironment) ByteBuf(org.apache.flink.shaded.netty4.io.netty.buffer.ByteBuf) Buffer(org.apache.flink.runtime.io.network.buffer.Buffer) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) PipelinedSubpartition(org.apache.flink.runtime.io.network.partition.PipelinedSubpartition) Assert.assertNull(org.junit.Assert.assertNull) Matchers.contains(org.hamcrest.Matchers.contains) Assert.assertFalse(org.junit.Assert.assertFalse) NoOpFileChannelManager(org.apache.flink.runtime.io.disk.NoOpFileChannelManager) PipelinedSubpartitionTest(org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionTest) FileChannelManagerImpl(org.apache.flink.runtime.io.disk.FileChannelManagerImpl) ResultPartitionManager(org.apache.flink.runtime.io.network.partition.ResultPartitionManager) ResultSubpartitionView(org.apache.flink.runtime.io.network.partition.ResultSubpartitionView) EmbeddedChannel(org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel) PartitionTestUtils.createPartition(org.apache.flink.runtime.io.network.partition.PartitionTestUtils.createPartition) ResultPartitionProvider(org.apache.flink.runtime.io.network.partition.ResultPartitionProvider) InputChannelID(org.apache.flink.runtime.io.network.partition.consumer.InputChannelID) TemporaryFolder(org.junit.rules.TemporaryFolder) Assert.assertEquals(org.junit.Assert.assertEquals) PipelinedSubpartition(org.apache.flink.runtime.io.network.partition.PipelinedSubpartition) EmbeddedChannel(org.apache.flink.shaded.netty4.io.netty.channel.embedded.EmbeddedChannel) PipelinedSubpartitionView(org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionView) InputChannelID(org.apache.flink.runtime.io.network.partition.consumer.InputChannelID) NoOpBufferAvailablityListener(org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener) ResultPartitionID(org.apache.flink.runtime.io.network.partition.ResultPartitionID) ResultPartitionProvider(org.apache.flink.runtime.io.network.partition.ResultPartitionProvider) Test(org.junit.Test) PipelinedSubpartitionTest(org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionTest)

Aggregations

IOException (java.io.IOException)2 ByteBuffer (java.nio.ByteBuffer)2 AtomicInteger (java.util.concurrent.atomic.AtomicInteger)2 Nullable (javax.annotation.Nullable)2 FileChannelManager (org.apache.flink.runtime.io.disk.FileChannelManager)2 FileChannelManagerImpl (org.apache.flink.runtime.io.disk.FileChannelManagerImpl)2 NoOpFileChannelManager (org.apache.flink.runtime.io.disk.NoOpFileChannelManager)2 NettyShuffleEnvironment (org.apache.flink.runtime.io.network.NettyShuffleEnvironment)2 NettyShuffleEnvironmentBuilder (org.apache.flink.runtime.io.network.NettyShuffleEnvironmentBuilder)2 NetworkSequenceViewReader (org.apache.flink.runtime.io.network.NetworkSequenceViewReader)2 Buffer (org.apache.flink.runtime.io.network.buffer.Buffer)2 BufferBuilderTestUtils.createEventBufferConsumer (org.apache.flink.runtime.io.network.buffer.BufferBuilderTestUtils.createEventBufferConsumer)2 BufferAvailabilityListener (org.apache.flink.runtime.io.network.partition.BufferAvailabilityListener)2 NoOpBufferAvailablityListener (org.apache.flink.runtime.io.network.partition.NoOpBufferAvailablityListener)2 NoOpResultSubpartitionView (org.apache.flink.runtime.io.network.partition.NoOpResultSubpartitionView)2 PartitionTestUtils.createPartition (org.apache.flink.runtime.io.network.partition.PartitionTestUtils.createPartition)2 PipelinedSubpartition (org.apache.flink.runtime.io.network.partition.PipelinedSubpartition)2 PipelinedSubpartitionTest (org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionTest)2 PipelinedSubpartitionView (org.apache.flink.runtime.io.network.partition.PipelinedSubpartitionView)2 ResultPartition (org.apache.flink.runtime.io.network.partition.ResultPartition)2