Examples with InputStatus org.apache.flink.core.io.InputStatus used on opensource projects

Example 1 with InputStatus

use of org.apache.flink.core.io.InputStatus in project flink by apache.

the class KafkaSourceReaderTest method testCommitOffsetsWithoutAliveFetchers.

// -----------------------------------------
@Test
void testCommitOffsetsWithoutAliveFetchers() throws Exception {
    final String groupId = "testCommitOffsetsWithoutAliveFetchers";
    try (KafkaSourceReader<Integer> reader = (KafkaSourceReader<Integer>) createReader(Boundedness.CONTINUOUS_UNBOUNDED, groupId)) {
        KafkaPartitionSplit split = new KafkaPartitionSplit(new TopicPartition(TOPIC, 0), 0, NUM_RECORDS_PER_SPLIT);
        reader.addSplits(Collections.singletonList(split));
        reader.notifyNoMoreSplits();
        ReaderOutput<Integer> output = new TestingReaderOutput<>();
        InputStatus status;
        do {
            status = reader.pollNext(output);
        } while (status != InputStatus.NOTHING_AVAILABLE);
        pollUntil(reader, output, () -> reader.getNumAliveFetchers() == 0, "The split fetcher did not exit before timeout.");
        reader.snapshotState(100L);
        reader.notifyCheckpointComplete(100L);
        // Due to a bug in KafkaConsumer, when the consumer closes, the offset commit callback
        // won't be fired, so the offsetsToCommit map won't be cleaned. To make the test
        // stable, we add a split whose starting offset is the log end offset, so the
        // split fetcher won't become idle and exit after commitOffsetAsync is invoked from
        // notifyCheckpointComplete().
        reader.addSplits(Collections.singletonList(new KafkaPartitionSplit(new TopicPartition(TOPIC, 0), NUM_RECORDS_PER_SPLIT)));
        pollUntil(reader, output, () -> reader.getOffsetsToCommit().isEmpty(), "The offset commit did not finish before timeout.");
    }
    // Verify the committed offsets.
    try (AdminClient adminClient = KafkaSourceTestEnv.getAdminClient()) {
        Map<TopicPartition, OffsetAndMetadata> committedOffsets = adminClient.listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata().get();
        assertThat(committedOffsets).hasSize(1);
        assertThat(committedOffsets.values()).extracting(OffsetAndMetadata::offset).allMatch(offset -> offset == NUM_RECORDS_PER_SPLIT);
    }
}

Example 2 with InputStatus

use of org.apache.flink.core.io.InputStatus in project flink by apache.

the class FileSourceHeavyThroughputTest method testHeavyThroughput.

// ------------------------------------------------------------------------
@Test
public void testHeavyThroughput() throws Exception {
    final Path path = new Path("testfs:///testpath");
    // 20 GB
    final long fileSize = 20L << 30;
    final FileSourceSplit split = new FileSourceSplit("testsplitId", path, 0, fileSize, 0, fileSize);
    testFs = TestingFileSystem.createForFileStatus(path.toUri().getScheme(), TestingFileSystem.TestFileStatus.forFileWithStream(path, fileSize, new GeneratingInputStream(fileSize)));
    testFs.register();
    final FileSource<byte[]> source = FileSource.forRecordStreamFormat(new ArrayReaderFormat(), path).build();
    final SourceReader<byte[], FileSourceSplit> reader = source.createReader(new NoOpReaderContext());
    reader.addSplits(Collections.singletonList(split));
    reader.notifyNoMoreSplits();
    final ReaderOutput<byte[]> out = new NoOpReaderOutput<>();
    InputStatus status;
    while ((status = reader.pollNext(out)) != InputStatus.END_OF_INPUT) {
        // if nothing is available currently, wait for more
        if (status == InputStatus.NOTHING_AVAILABLE) {
            reader.isAvailable().get();
        }
    }
}

Example 3 with InputStatus

use of org.apache.flink.core.io.InputStatus in project flink by apache.

the class PulsarSourceReaderTestBase method assigningEmptySplits.

@TestTemplate
void assigningEmptySplits(PulsarSourceReaderBase<Integer> reader, Boundedness boundedness, String topicName) throws Exception {
    final PulsarPartitionSplit emptySplit = createPartitionSplit(topicName, 0, Boundedness.CONTINUOUS_UNBOUNDED, MessageId.latest);
    reader.addSplits(Collections.singletonList(emptySplit));
    TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
    InputStatus status = reader.pollNext(output);
    assertThat(status).isEqualTo(InputStatus.NOTHING_AVAILABLE);
    reader.close();
}

Example 4 with InputStatus

use of org.apache.flink.core.io.InputStatus in project flink by apache.

the class PulsarOrderedSourceReaderTest method offsetCommitOnCheckpointComplete.

@TestTemplate
void offsetCommitOnCheckpointComplete(PulsarSourceReaderBase<Integer> baseReader, Boundedness boundedness, String topicName) throws Exception {
    PulsarOrderedSourceReader<Integer> reader = (PulsarOrderedSourceReader<Integer>) baseReader;
    // consume more than 1 partition
    reader.addSplits(createPartitionSplits(topicName, DEFAULT_PARTITIONS, Boundedness.CONTINUOUS_UNBOUNDED));
    reader.notifyNoMoreSplits();
    TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
    long checkpointId = 0;
    int emptyResultTime = 0;
    InputStatus status;
    do {
        checkpointId++;
        status = reader.pollNext(output);
        // Create a checkpoint for each message consumption, but not complete them.
        reader.snapshotState(checkpointId);
        // the first couple of pollNext() might return NOTHING_AVAILABLE before data appears
        if (InputStatus.NOTHING_AVAILABLE == status) {
            emptyResultTime++;
            sleepUninterruptibly(1, TimeUnit.SECONDS);
        }
    } while (emptyResultTime < MAX_EMPTY_POLLING_TIMES && status != InputStatus.END_OF_INPUT && output.getEmittedRecords().size() < NUM_RECORDS_PER_PARTITION * DEFAULT_PARTITIONS);
    // The completion of the last checkpoint should subsume all previous checkpoints.
    assertThat(reader.cursorsToCommit).hasSize((int) checkpointId);
    long lastCheckpointId = checkpointId;
    // notify checkpoint complete and expect all cursors committed
    assertThatCode(() -> reader.notifyCheckpointComplete(lastCheckpointId)).doesNotThrowAnyException();
    assertThat(reader.cursorsToCommit).isEmpty();
    // Verify the committed offsets.
    reader.close();
    for (int i = 0; i < DEFAULT_PARTITIONS; i++) {
        verifyAllMessageAcknowledged(NUM_RECORDS_PER_PARTITION, TopicNameUtils.topicNameWithPartition(topicName, i));
    }
}

Example 5 with InputStatus

use of org.apache.flink.core.io.InputStatus in project flink by apache.

the class HybridSourceReaderTest method testReader.

@Test
public void testReader() throws Exception {
    TestingReaderContext readerContext = new TestingReaderContext();
    TestingReaderOutput<Integer> readerOutput = new TestingReaderOutput<>();
    MockBaseSource source = new MockBaseSource(1, 1, Boundedness.BOUNDED);
    // 2 underlying readers to exercise switch
    SourceReader<Integer, MockSourceSplit> mockSplitReader1 = source.createReader(readerContext);
    SourceReader<Integer, MockSourceSplit> mockSplitReader2 = source.createReader(readerContext);
    HybridSourceReader<Integer> reader = new HybridSourceReader<>(readerContext);
    Assert.assertThat(readerContext.getSentEvents(), Matchers.emptyIterable());
    reader.start();
    assertAndClearSourceReaderFinishedEvent(readerContext, -1);
    Assert.assertNull(currentReader(reader));
    Assert.assertEquals(InputStatus.NOTHING_AVAILABLE, reader.pollNext(readerOutput));
    Source source1 = new MockSource(null, 0) {

        @Override
        public SourceReader<Integer, MockSourceSplit> createReader(SourceReaderContext readerContext) {
            return mockSplitReader1;
        }
    };
    reader.handleSourceEvents(new SwitchSourceEvent(0, source1, false));
    MockSourceSplit mockSplit = new MockSourceSplit(0, 0, 1);
    mockSplit.addRecord(0);
    SwitchedSources switchedSources = new SwitchedSources();
    switchedSources.put(0, source);
    HybridSourceSplit hybridSplit = HybridSourceSplit.wrapSplit(mockSplit, 0, switchedSources);
    reader.addSplits(Collections.singletonList(hybridSplit));
    // drain splits
    InputStatus status = reader.pollNext(readerOutput);
    while (readerOutput.getEmittedRecords().isEmpty() || status == InputStatus.MORE_AVAILABLE) {
        status = reader.pollNext(readerOutput);
        Thread.sleep(10);
    }
    Assert.assertThat(readerOutput.getEmittedRecords(), Matchers.contains(0));
    reader.pollNext(readerOutput);
    Assert.assertEquals("before notifyNoMoreSplits", InputStatus.NOTHING_AVAILABLE, reader.pollNext(readerOutput));
    reader.notifyNoMoreSplits();
    reader.pollNext(readerOutput);
    assertAndClearSourceReaderFinishedEvent(readerContext, 0);
    Assert.assertEquals("reader before switch source event", mockSplitReader1, currentReader(reader));
    Source source2 = new MockSource(null, 0) {

        @Override
        public SourceReader<Integer, MockSourceSplit> createReader(SourceReaderContext readerContext) {
            return mockSplitReader2;
        }
    };
    reader.handleSourceEvents(new SwitchSourceEvent(1, source2, true));
    Assert.assertEquals("reader after switch source event", mockSplitReader2, currentReader(reader));
    reader.notifyNoMoreSplits();
    Assert.assertEquals("reader 1 after notifyNoMoreSplits", InputStatus.END_OF_INPUT, reader.pollNext(readerOutput));
    reader.close();
}