Examples with TestingReaderOutput org.apache.flink.connector.testutils.source.reader.TestingReaderOutput used on opensource projects

Example 16 with TestingReaderOutput

use of org.apache.flink.connector.testutils.source.reader.TestingReaderOutput in project flink-mirror by flink-ci.

the class PulsarOrderedSourceReaderTest method offsetCommitOnCheckpointComplete.

@TestTemplate
void offsetCommitOnCheckpointComplete(PulsarSourceReaderBase<Integer> baseReader, Boundedness boundedness, String topicName) throws Exception {
    PulsarOrderedSourceReader<Integer> reader = (PulsarOrderedSourceReader<Integer>) baseReader;
    // consume more than 1 partition
    reader.addSplits(createPartitionSplits(topicName, DEFAULT_PARTITIONS, Boundedness.CONTINUOUS_UNBOUNDED));
    reader.notifyNoMoreSplits();
    TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
    long checkpointId = 0;
    int emptyResultTime = 0;
    InputStatus status;
    do {
        checkpointId++;
        status = reader.pollNext(output);
        // Create a checkpoint for each message consumption, but not complete them.
        reader.snapshotState(checkpointId);
        // the first couple of pollNext() might return NOTHING_AVAILABLE before data appears
        if (InputStatus.NOTHING_AVAILABLE == status) {
            emptyResultTime++;
            sleepUninterruptibly(1, TimeUnit.SECONDS);
        }
    } while (emptyResultTime < MAX_EMPTY_POLLING_TIMES && status != InputStatus.END_OF_INPUT && output.getEmittedRecords().size() < NUM_RECORDS_PER_PARTITION * DEFAULT_PARTITIONS);
    // The completion of the last checkpoint should subsume all previous checkpoints.
    assertThat(reader.cursorsToCommit).hasSize((int) checkpointId);
    long lastCheckpointId = checkpointId;
    // notify checkpoint complete and expect all cursors committed
    assertThatCode(() -> reader.notifyCheckpointComplete(lastCheckpointId)).doesNotThrowAnyException();
    assertThat(reader.cursorsToCommit).isEmpty();
    // Verify the committed offsets.
    reader.close();
    for (int i = 0; i < DEFAULT_PARTITIONS; i++) {
        verifyAllMessageAcknowledged(NUM_RECORDS_PER_PARTITION, TopicNameUtils.topicNameWithPartition(topicName, i));
    }
}

Example 17 with TestingReaderOutput

use of org.apache.flink.connector.testutils.source.reader.TestingReaderOutput in project flink-mirror by flink-ci.

the class KafkaSourceReaderTest method testCommitOffsetsWithoutAliveFetchers.

// -----------------------------------------
@Test
void testCommitOffsetsWithoutAliveFetchers() throws Exception {
    final String groupId = "testCommitOffsetsWithoutAliveFetchers";
    try (KafkaSourceReader<Integer> reader = (KafkaSourceReader<Integer>) createReader(Boundedness.CONTINUOUS_UNBOUNDED, groupId)) {
        KafkaPartitionSplit split = new KafkaPartitionSplit(new TopicPartition(TOPIC, 0), 0, NUM_RECORDS_PER_SPLIT);
        reader.addSplits(Collections.singletonList(split));
        reader.notifyNoMoreSplits();
        ReaderOutput<Integer> output = new TestingReaderOutput<>();
        InputStatus status;
        do {
            status = reader.pollNext(output);
        } while (status != InputStatus.NOTHING_AVAILABLE);
        pollUntil(reader, output, () -> reader.getNumAliveFetchers() == 0, "The split fetcher did not exit before timeout.");
        reader.snapshotState(100L);
        reader.notifyCheckpointComplete(100L);
        // Due to a bug in KafkaConsumer, when the consumer closes, the offset commit callback
        // won't be fired, so the offsetsToCommit map won't be cleaned. To make the test
        // stable, we add a split whose starting offset is the log end offset, so the
        // split fetcher won't become idle and exit after commitOffsetAsync is invoked from
        // notifyCheckpointComplete().
        reader.addSplits(Collections.singletonList(new KafkaPartitionSplit(new TopicPartition(TOPIC, 0), NUM_RECORDS_PER_SPLIT)));
        pollUntil(reader, output, () -> reader.getOffsetsToCommit().isEmpty(), "The offset commit did not finish before timeout.");
    }
    // Verify the committed offsets.
    try (AdminClient adminClient = KafkaSourceTestEnv.getAdminClient()) {
        Map<TopicPartition, OffsetAndMetadata> committedOffsets = adminClient.listConsumerGroupOffsets(groupId).partitionsToOffsetAndMetadata().get();
        assertThat(committedOffsets).hasSize(1);
        assertThat(committedOffsets.values()).extracting(OffsetAndMetadata::offset).allMatch(offset -> offset == NUM_RECORDS_PER_SPLIT);
    }
}

Example 18 with TestingReaderOutput

use of org.apache.flink.connector.testutils.source.reader.TestingReaderOutput in project flink-mirror by flink-ci.

the class KafkaSourceReaderTest method testKafkaSourceMetrics.

@Test
void testKafkaSourceMetrics() throws Exception {
    final MetricListener metricListener = new MetricListener();
    final String groupId = "testKafkaSourceMetrics";
    final TopicPartition tp0 = new TopicPartition(TOPIC, 0);
    final TopicPartition tp1 = new TopicPartition(TOPIC, 1);
    try (KafkaSourceReader<Integer> reader = (KafkaSourceReader<Integer>) createReader(Boundedness.CONTINUOUS_UNBOUNDED, groupId, metricListener.getMetricGroup())) {
        KafkaPartitionSplit split0 = new KafkaPartitionSplit(tp0, KafkaPartitionSplit.EARLIEST_OFFSET);
        KafkaPartitionSplit split1 = new KafkaPartitionSplit(tp1, KafkaPartitionSplit.EARLIEST_OFFSET);
        reader.addSplits(Arrays.asList(split0, split1));
        TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
        pollUntil(reader, output, () -> output.getEmittedRecords().size() == NUM_RECORDS_PER_SPLIT * 2, String.format("Failed to poll %d records until timeout", NUM_RECORDS_PER_SPLIT * 2));
        // Metric "records-consumed-total" of KafkaConsumer should be NUM_RECORDS_PER_SPLIT
        assertThat(getKafkaConsumerMetric("records-consumed-total", metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT * 2);
        // Current consuming offset should be NUM_RECORD_PER_SPLIT - 1
        assertThat(getCurrentOffsetMetric(tp0, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT - 1);
        assertThat(getCurrentOffsetMetric(tp1, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT - 1);
        // No offset is committed till now
        assertThat(getCommittedOffsetMetric(tp0, metricListener)).isEqualTo(INITIAL_OFFSET);
        assertThat(getCommittedOffsetMetric(tp1, metricListener)).isEqualTo(INITIAL_OFFSET);
        // Trigger offset commit
        final long checkpointId = 15213L;
        reader.snapshotState(checkpointId);
        waitUtil(() -> {
            try {
                reader.notifyCheckpointComplete(checkpointId);
            } catch (Exception e) {
                throw new RuntimeException("Failed to notify checkpoint complete to reader", e);
            }
            return reader.getOffsetsToCommit().isEmpty();
        }, Duration.ofSeconds(60), Duration.ofSeconds(1), String.format("Offsets are not committed successfully. Dangling offsets: %s", reader.getOffsetsToCommit()));
        // Metric "commit-total" of KafkaConsumer should be greater than 0
        // It's hard to know the exactly number of commit because of the retry
        MatcherAssert.assertThat(getKafkaConsumerMetric("commit-total", metricListener), Matchers.greaterThan(0L));
        // Committed offset should be NUM_RECORD_PER_SPLIT
        assertThat(getCommittedOffsetMetric(tp0, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT);
        assertThat(getCommittedOffsetMetric(tp1, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT);
        // Number of successful commits should be greater than 0
        final Optional<Counter> commitsSucceeded = metricListener.getCounter(KAFKA_SOURCE_READER_METRIC_GROUP, COMMITS_SUCCEEDED_METRIC_COUNTER);
        assertThat(commitsSucceeded).isPresent();
        MatcherAssert.assertThat(commitsSucceeded.get().getCount(), Matchers.greaterThan(0L));
    }
}

Example 19 with TestingReaderOutput

use of org.apache.flink.connector.testutils.source.reader.TestingReaderOutput in project flink by splunk.

the class KafkaSourceReaderTest method testKafkaSourceMetrics.

@Test
void testKafkaSourceMetrics() throws Exception {
    final MetricListener metricListener = new MetricListener();
    final String groupId = "testKafkaSourceMetrics";
    final TopicPartition tp0 = new TopicPartition(TOPIC, 0);
    final TopicPartition tp1 = new TopicPartition(TOPIC, 1);
    try (KafkaSourceReader<Integer> reader = (KafkaSourceReader<Integer>) createReader(Boundedness.CONTINUOUS_UNBOUNDED, groupId, metricListener.getMetricGroup())) {
        KafkaPartitionSplit split0 = new KafkaPartitionSplit(tp0, KafkaPartitionSplit.EARLIEST_OFFSET);
        KafkaPartitionSplit split1 = new KafkaPartitionSplit(tp1, KafkaPartitionSplit.EARLIEST_OFFSET);
        reader.addSplits(Arrays.asList(split0, split1));
        TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
        pollUntil(reader, output, () -> output.getEmittedRecords().size() == NUM_RECORDS_PER_SPLIT * 2, String.format("Failed to poll %d records until timeout", NUM_RECORDS_PER_SPLIT * 2));
        // Metric "records-consumed-total" of KafkaConsumer should be NUM_RECORDS_PER_SPLIT
        assertThat(getKafkaConsumerMetric("records-consumed-total", metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT * 2);
        // Current consuming offset should be NUM_RECORD_PER_SPLIT - 1
        assertThat(getCurrentOffsetMetric(tp0, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT - 1);
        assertThat(getCurrentOffsetMetric(tp1, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT - 1);
        // No offset is committed till now
        assertThat(getCommittedOffsetMetric(tp0, metricListener)).isEqualTo(INITIAL_OFFSET);
        assertThat(getCommittedOffsetMetric(tp1, metricListener)).isEqualTo(INITIAL_OFFSET);
        // Trigger offset commit
        final long checkpointId = 15213L;
        reader.snapshotState(checkpointId);
        waitUtil(() -> {
            try {
                reader.notifyCheckpointComplete(checkpointId);
            } catch (Exception e) {
                throw new RuntimeException("Failed to notify checkpoint complete to reader", e);
            }
            return reader.getOffsetsToCommit().isEmpty();
        }, Duration.ofSeconds(60), Duration.ofSeconds(1), String.format("Offsets are not committed successfully. Dangling offsets: %s", reader.getOffsetsToCommit()));
        // Metric "commit-total" of KafkaConsumer should be greater than 0
        // It's hard to know the exactly number of commit because of the retry
        MatcherAssert.assertThat(getKafkaConsumerMetric("commit-total", metricListener), Matchers.greaterThan(0L));
        // Committed offset should be NUM_RECORD_PER_SPLIT
        assertThat(getCommittedOffsetMetric(tp0, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT);
        assertThat(getCommittedOffsetMetric(tp1, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT);
        // Number of successful commits should be greater than 0
        final Optional<Counter> commitsSucceeded = metricListener.getCounter(KAFKA_SOURCE_READER_METRIC_GROUP, COMMITS_SUCCEEDED_METRIC_COUNTER);
        assertThat(commitsSucceeded).isPresent();
        MatcherAssert.assertThat(commitsSucceeded.get().getCount(), Matchers.greaterThan(0L));
    }
}

Example 20 with TestingReaderOutput

use of org.apache.flink.connector.testutils.source.reader.TestingReaderOutput in project flink by splunk.

the class HybridSourceReaderTest method testDefaultMethodDelegation.

@Test
public void testDefaultMethodDelegation() throws Exception {
    TestingReaderContext readerContext = new TestingReaderContext();
    TestingReaderOutput<Integer> readerOutput = new TestingReaderOutput<>();
    MockBaseSource source = new MockBaseSource(1, 1, Boundedness.BOUNDED) {

        @Override
        public SourceReader<Integer, MockSourceSplit> createReader(SourceReaderContext readerContext) {
            return Mockito.spy(super.createReader(readerContext));
        }
    };
    HybridSourceReader<Integer> reader = new HybridSourceReader<>(readerContext);
    reader.start();
    assertAndClearSourceReaderFinishedEvent(readerContext, -1);
    reader.handleSourceEvents(new SwitchSourceEvent(0, source, false));
    SourceReader<Integer, MockSourceSplit> underlyingReader = currentReader(reader);
    reader.notifyCheckpointComplete(1);
    Mockito.verify(underlyingReader).notifyCheckpointComplete(1);
    reader.notifyCheckpointAborted(1);
    Mockito.verify(underlyingReader).notifyCheckpointAborted(1);
    reader.close();
}