Examples with BufferConsumer org.apache.flink.runtime.io.network.buffer.BufferConsumer used on opensource projects

Example 16 with BufferConsumer

use of org.apache.flink.runtime.io.network.buffer.BufferConsumer in project flink by apache.

the class ChannelStateSerializerImplTest method testReadToBufferBuilder.

@Test
public void testReadToBufferBuilder() throws IOException {
    byte[] data = generateData(100);
    BufferBuilder bufferBuilder = new BufferBuilder(MemorySegmentFactory.allocateUnpooledSegment(data.length, null), FreeingBufferRecycler.INSTANCE);
    BufferConsumer bufferConsumer = bufferBuilder.createBufferConsumer();
    new ChannelStateSerializerImpl().readData(new ByteArrayInputStream(data), wrap(bufferBuilder), Integer.MAX_VALUE);
    assertFalse(bufferBuilder.isFinished());
    bufferBuilder.finish();
    Buffer buffer = bufferConsumer.build();
    assertEquals(data.length, buffer.readableBytes());
    byte[] actual = new byte[buffer.readableBytes()];
    buffer.asByteBuf().readBytes(actual);
    assertArrayEquals(data, actual);
}

Example 17 with BufferConsumer

use of org.apache.flink.runtime.io.network.buffer.BufferConsumer in project flink by apache.

the class PipelinedSubpartitionWithReadViewTest method testBarrierOvertaking.

@Test
public void testBarrierOvertaking() throws Exception {
    final RecordingChannelStateWriter channelStateWriter = new RecordingChannelStateWriter();
    subpartition.setChannelStateWriter(channelStateWriter);
    subpartition.add(createFilledFinishedBufferConsumer(1));
    assertEquals(0, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(2));
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    BufferConsumer eventBuffer = EventSerializer.toBufferConsumer(EndOfSuperstepEvent.INSTANCE, false);
    subpartition.add(eventBuffer);
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(4));
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(0, availablityListener.getNumPriorityEvents());
    CheckpointOptions options = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, new CheckpointStorageLocationReference(new byte[] { 0, 1, 2 }));
    channelStateWriter.start(0, options);
    BufferConsumer barrierBuffer = EventSerializer.toBufferConsumer(new CheckpointBarrier(0, 0, options), true);
    subpartition.add(barrierBuffer);
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(1, availablityListener.getNumPriorityEvents());
    final List<Buffer> inflight = channelStateWriter.getAddedOutput().get(subpartition.getSubpartitionInfo());
    assertEquals(Arrays.asList(1, 2, 4), inflight.stream().map(Buffer::getSize).collect(Collectors.toList()));
    inflight.forEach(Buffer::recycleBuffer);
    assertNextEvent(readView, barrierBuffer.getWrittenBytes(), CheckpointBarrier.class, true, 2, false, true);
    assertNextBuffer(readView, 1, true, 1, false, true);
    assertNextBuffer(readView, 2, true, 0, true, true);
    assertNextEvent(readView, eventBuffer.getWrittenBytes(), EndOfSuperstepEvent.class, false, 0, false, true);
    assertNextBuffer(readView, 4, false, 0, false, true);
    assertNoNextBuffer(readView);
}

Example 18 with BufferConsumer

use of org.apache.flink.runtime.io.network.buffer.BufferConsumer in project flink by apache.

the class PipelinedSubpartitionWithReadViewTest method testAvailabilityAfterPriority.

@Test
public void testAvailabilityAfterPriority() throws Exception {
    subpartition.setChannelStateWriter(ChannelStateWriter.NO_OP);
    CheckpointOptions options = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, new CheckpointStorageLocationReference(new byte[] { 0, 1, 2 }));
    BufferConsumer barrierBuffer = EventSerializer.toBufferConsumer(new CheckpointBarrier(0, 0, options), true);
    subpartition.add(barrierBuffer);
    assertEquals(1, availablityListener.getNumNotifications());
    assertEquals(1, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(1));
    assertEquals(2, availablityListener.getNumNotifications());
    assertEquals(1, availablityListener.getNumPriorityEvents());
    subpartition.add(createFilledFinishedBufferConsumer(2));
    assertEquals(2, availablityListener.getNumNotifications());
    assertEquals(1, availablityListener.getNumPriorityEvents());
    assertNextEvent(readView, barrierBuffer.getWrittenBytes(), CheckpointBarrier.class, true, 1, false, true);
    assertNextBuffer(readView, 1, false, 0, false, true);
    assertNextBuffer(readView, 2, false, 0, false, true);
    assertNoNextBuffer(readView);
}

Example 19 with BufferConsumer

use of org.apache.flink.runtime.io.network.buffer.BufferConsumer in project flink by apache.

the class PipelinedSubpartitionTest method testCleanupReleasedPartition.

/**
 * Tests cleanup of {@link PipelinedSubpartition#release()}.
 *
 * @param createView whether the partition should have a view attached to it (<tt>true</tt>) or
 *     not (<tt>false</tt>)
 */
private void testCleanupReleasedPartition(boolean createView) throws Exception {
    PipelinedSubpartition partition = createSubpartition();
    BufferConsumer buffer1 = createFilledFinishedBufferConsumer(4096);
    BufferConsumer buffer2 = createFilledFinishedBufferConsumer(4096);
    boolean buffer1Recycled;
    boolean buffer2Recycled;
    try {
        partition.add(buffer1);
        partition.add(buffer2);
        assertEquals(2, partition.getNumberOfQueuedBuffers());
        // create the read view first
        ResultSubpartitionView view = null;
        if (createView) {
            view = partition.createReadView(new NoOpBufferAvailablityListener());
        }
        partition.release();
        assertEquals(0, partition.getNumberOfQueuedBuffers());
        assertTrue(partition.isReleased());
        if (createView) {
            assertTrue(view.isReleased());
        }
        assertTrue(buffer1.isRecycled());
    } finally {
        buffer1Recycled = buffer1.isRecycled();
        if (!buffer1Recycled) {
            buffer1.close();
        }
        buffer2Recycled = buffer2.isRecycled();
        if (!buffer2Recycled) {
            buffer2.close();
        }
    }
    if (!buffer1Recycled) {
        Assert.fail("buffer 1 not recycled");
    }
    if (!buffer2Recycled) {
        Assert.fail("buffer 2 not recycled");
    }
    assertEquals(2, partition.getTotalNumberOfBuffersUnsafe());
    // buffer data is never consumed
    assertEquals(0, partition.getTotalNumberOfBytesUnsafe());
}

Example 20 with BufferConsumer

use of org.apache.flink.runtime.io.network.buffer.BufferConsumer in project flink by apache.

the class LocalInputChannelTest method testNoDataPersistedAfterReceivingAlignedBarrier.

@Test
public void testNoDataPersistedAfterReceivingAlignedBarrier() throws Exception {
    CheckpointBarrier barrier = new CheckpointBarrier(1L, 0L, CheckpointOptions.alignedWithTimeout(CheckpointType.CHECKPOINT, getDefault(), 123L));
    BufferConsumer barrierHolder = EventSerializer.toBufferConsumer(barrier, false);
    BufferConsumer data = createFilledFinishedBufferConsumer(1);
    RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    LocalInputChannel channel = InputChannelBuilder.newBuilder().setPartitionManager(new TestingResultPartitionManager(InputChannelTestUtils.createResultSubpartitionView(barrierHolder, data))).setStateWriter(stateWriter).buildLocalChannel(new SingleInputGateBuilder().build());
    channel.requestSubpartition();
    // pull AC barrier
    channel.getNextBuffer();
    // pretend that alignment timed out
    stateWriter.start(barrier.getId(), barrier.getCheckpointOptions());
    channel.checkpointStarted(barrier);
    // pull data
    channel.getNextBuffer();
    assertTrue("no data should be persisted after receiving a barrier", stateWriter.getAddedInput().isEmpty());
}