Examples with PipelinedResultPartition org.apache.flink.runtime.io.network.partition.PipelinedResultPartition used on opensource projects

Example 1 with PipelinedResultPartition

use of org.apache.flink.runtime.io.network.partition.PipelinedResultPartition in project flink by apache.

the class LocalInputChannelTest method testCheckpointingInflightData.

@Test
public void testCheckpointingInflightData() throws Exception {
    SingleInputGate inputGate = new SingleInputGateBuilder().build();
    PipelinedResultPartition parent = (PipelinedResultPartition) PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED, NoOpFileChannelManager.INSTANCE);
    ResultSubpartition subpartition = parent.getAllPartitions()[0];
    ResultSubpartitionView subpartitionView = subpartition.createReadView(() -> {
    });
    TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(subpartitionView);
    final RecordingChannelStateWriter stateWriter = new RecordingChannelStateWriter();
    LocalInputChannel channel = createLocalInputChannel(inputGate, partitionManager, 0, 0, b -> b.setStateWriter(stateWriter));
    inputGate.setInputChannels(channel);
    channel.requestSubpartition();
    final CheckpointStorageLocationReference location = getDefault();
    CheckpointOptions options = CheckpointOptions.unaligned(CheckpointType.CHECKPOINT, location);
    stateWriter.start(0, options);
    final CheckpointBarrier barrier = new CheckpointBarrier(0, 123L, options);
    channel.checkpointStarted(barrier);
    // add 1 buffer before barrier and 1 buffer afterwards. Only the first buffer should be
    // written.
    subpartition.add(createFilledFinishedBufferConsumer(1));
    assertTrue(channel.getNextBuffer().isPresent());
    subpartition.add(EventSerializer.toBufferConsumer(barrier, true));
    assertTrue(channel.getNextBuffer().isPresent());
    subpartition.add(createFilledFinishedBufferConsumer(2));
    assertTrue(channel.getNextBuffer().isPresent());
    assertArrayEquals(stateWriter.getAddedInput().get(channel.getChannelInfo()).stream().mapToInt(Buffer::getSize).toArray(), new int[] { 1 });
}

Example 2 with PipelinedResultPartition

use of org.apache.flink.runtime.io.network.partition.PipelinedResultPartition in project flink by apache.

the class StreamTaskFinalCheckpointsTest method testNotWaitingForAllRecordsProcessedIfCheckpointNotEnabled.

@Test
public void testNotWaitingForAllRecordsProcessedIfCheckpointNotEnabled() throws Exception {
    ResultPartitionWriter[] partitionWriters = new ResultPartitionWriter[2];
    try {
        for (int i = 0; i < partitionWriters.length; ++i) {
            partitionWriters[i] = PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED_BOUNDED);
            partitionWriters[i].setup();
        }
        try (StreamTaskMailboxTestHarness<String> testHarness = new StreamTaskMailboxTestHarnessBuilder<>(OneInputStreamTask::new, STRING_TYPE_INFO).modifyStreamConfig(config -> config.setCheckpointingEnabled(false)).addInput(STRING_TYPE_INFO).addAdditionalOutput(partitionWriters).setupOperatorChain(new EmptyOperator()).finishForSingletonOperatorChain(StringSerializer.INSTANCE).build()) {
            testHarness.endInput();
            // In this case the result partition should not emit EndOfUserRecordsEvent.
            for (ResultPartitionWriter writer : partitionWriters) {
                assertEquals(0, ((PipelinedResultPartition) writer).getNumberOfQueuedBuffers());
            }
        }
    } finally {
        for (ResultPartitionWriter writer : partitionWriters) {
            if (writer != null) {
                writer.close();
            }
        }
    }
}

Example 3 with PipelinedResultPartition

use of org.apache.flink.runtime.io.network.partition.PipelinedResultPartition in project flink by apache.

the class LocalInputChannelTest method testAnnounceNewBufferSize.

@Test
public void testAnnounceNewBufferSize() throws IOException, InterruptedException {
    // given: Configured LocalInputChannel and pipelined subpartition.
    PipelinedResultPartition parent = (PipelinedResultPartition) new ResultPartitionBuilder().setResultPartitionType(ResultPartitionType.PIPELINED).setFileChannelManager(NoOpFileChannelManager.INSTANCE).setNumberOfSubpartitions(2).build();
    ResultSubpartition subpartition0 = parent.getAllPartitions()[0];
    ResultSubpartition subpartition1 = parent.getAllPartitions()[1];
    LocalInputChannel channel0 = createLocalInputChannel(new SingleInputGateBuilder().build(), new TestingResultPartitionManager(subpartition0.createReadView(() -> {
    })));
    LocalInputChannel channel1 = createLocalInputChannel(new SingleInputGateBuilder().build(), new TestingResultPartitionManager(subpartition1.createReadView(() -> {
    })));
    channel0.requestSubpartition();
    channel1.requestSubpartition();
    // and: Preferable buffer size is default value.
    assertEquals(Integer.MAX_VALUE, subpartition0.add(createFilledFinishedBufferConsumer(16)));
    assertEquals(Integer.MAX_VALUE, subpartition1.add(createFilledFinishedBufferConsumer(16)));
    // when: Announce the different buffer size for different channels via LocalInputChannel.
    channel0.announceBufferSize(9);
    channel1.announceBufferSize(20);
    // then: The corresponded subpartitions have the new size.
    assertEquals(9, subpartition0.add(createFilledFinishedBufferConsumer(16)));
    assertEquals(20, subpartition1.add(createFilledFinishedBufferConsumer(16)));
}

Example 4 with PipelinedResultPartition

use of org.apache.flink.runtime.io.network.partition.PipelinedResultPartition in project flink by apache.

the class LocalInputChannelTest method testEnqueueAvailableChannelWhenResuming.

@Test
public void testEnqueueAvailableChannelWhenResuming() throws IOException, InterruptedException {
    PipelinedResultPartition parent = (PipelinedResultPartition) PartitionTestUtils.createPartition(ResultPartitionType.PIPELINED, NoOpFileChannelManager.INSTANCE);
    ResultSubpartition subpartition = parent.getAllPartitions()[0];
    ResultSubpartitionView subpartitionView = subpartition.createReadView(() -> {
    });
    TestingResultPartitionManager partitionManager = new TestingResultPartitionManager(subpartitionView);
    LocalInputChannel channel = createLocalInputChannel(new SingleInputGateBuilder().build(), partitionManager);
    channel.requestSubpartition();
    // Block the subpartition
    subpartition.add(EventSerializer.toBufferConsumer(new CheckpointBarrier(1, 1, CheckpointOptions.forCheckpointWithDefaultLocation()), false));
    assertTrue(channel.getNextBuffer().isPresent());
    // Add more data
    subpartition.add(createFilledFinishedBufferConsumer(4096));
    subpartition.flush();
    // No buffer since the subpartition is blocked.
    assertFalse(channel.inputGate.pollNext().isPresent());
    // Resumption makes the subpartition available.
    channel.resumeConsumption();
    Optional<BufferOrEvent> nextBuffer = channel.inputGate.pollNext();
    assertTrue(nextBuffer.isPresent());
    assertTrue(nextBuffer.get().isBuffer());
}