Examples with KafkaPartitionSplit org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit used on opensource projects

Example 21 with KafkaPartitionSplit

use of org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit in project flink by apache.

the class KafkaPartitionSplitReaderTest method testAssignEmptySplit.

@Test
public void testAssignEmptySplit() throws Exception {
    KafkaPartitionSplitReader reader = createReader();
    final KafkaPartitionSplit normalSplit = new KafkaPartitionSplit(new TopicPartition(TOPIC1, 0), KafkaPartitionSplit.EARLIEST_OFFSET, KafkaPartitionSplit.NO_STOPPING_OFFSET);
    final KafkaPartitionSplit emptySplit = new KafkaPartitionSplit(new TopicPartition(TOPIC2, 0), KafkaPartitionSplit.LATEST_OFFSET, KafkaPartitionSplit.LATEST_OFFSET);
    reader.handleSplitsChanges(new SplitsAddition<>(Arrays.asList(normalSplit, emptySplit)));
    // Fetch and check empty splits is added to finished splits
    RecordsWithSplitIds<ConsumerRecord<byte[], byte[]>> recordsWithSplitIds = reader.fetch();
    assertTrue(recordsWithSplitIds.finishedSplits().contains(emptySplit.splitId()));
    // Assign another valid split to avoid consumer.poll() blocking
    final KafkaPartitionSplit anotherNormalSplit = new KafkaPartitionSplit(new TopicPartition(TOPIC1, 1), KafkaPartitionSplit.EARLIEST_OFFSET, KafkaPartitionSplit.NO_STOPPING_OFFSET);
    reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(anotherNormalSplit)));
    // Fetch again and check empty split set is cleared
    recordsWithSplitIds = reader.fetch();
    assertTrue(recordsWithSplitIds.finishedSplits().isEmpty());
}

Example 22 with KafkaPartitionSplit

use of org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit in project flink by apache.

the class KafkaPartitionSplitReaderTest method testNumBytesInCounter.

@Test
public void testNumBytesInCounter() throws Exception {
    final OperatorMetricGroup operatorMetricGroup = UnregisteredMetricGroups.createUnregisteredOperatorMetricGroup();
    final Counter numBytesInCounter = operatorMetricGroup.getIOMetricGroup().getNumBytesInCounter();
    KafkaPartitionSplitReader reader = createReader(new Properties(), InternalSourceReaderMetricGroup.wrap(operatorMetricGroup));
    // Add a split
    reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(new KafkaPartitionSplit(new TopicPartition(TOPIC1, 0), 0L))));
    reader.fetch();
    final long latestNumBytesIn = numBytesInCounter.getCount();
    // Since it's hard to know the exact number of bytes consumed, we just check if it is
    // greater than 0
    assertThat(latestNumBytesIn, Matchers.greaterThan(0L));
    // Add another split
    reader.handleSplitsChanges(new SplitsAddition<>(Collections.singletonList(new KafkaPartitionSplit(new TopicPartition(TOPIC2, 0), 0L))));
    reader.fetch();
    // We just check if numBytesIn is increasing
    assertThat(numBytesInCounter.getCount(), Matchers.greaterThan(latestNumBytesIn));
}

Example 23 with KafkaPartitionSplit

use of org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit in project flink by apache.

the class KafkaSourceReaderTest method testAssigningEmptySplits.

@Test
void testAssigningEmptySplits() throws Exception {
    // Normal split with NUM_RECORDS_PER_SPLIT records
    final KafkaPartitionSplit normalSplit = new KafkaPartitionSplit(new TopicPartition(TOPIC, 0), 0, KafkaPartitionSplit.LATEST_OFFSET);
    // Empty split with no record
    final KafkaPartitionSplit emptySplit = new KafkaPartitionSplit(new TopicPartition(TOPIC, 1), NUM_RECORDS_PER_SPLIT, NUM_RECORDS_PER_SPLIT);
    // Split finished hook for listening finished splits
    final Set<String> finishedSplits = new HashSet<>();
    final Consumer<Collection<String>> splitFinishedHook = finishedSplits::addAll;
    try (final KafkaSourceReader<Integer> reader = (KafkaSourceReader<Integer>) createReader(Boundedness.BOUNDED, "KafkaSourceReaderTestGroup", new TestingReaderContext(), splitFinishedHook)) {
        reader.addSplits(Arrays.asList(normalSplit, emptySplit));
        pollUntil(reader, new TestingReaderOutput<>(), () -> reader.getNumAliveFetchers() == 0, "The split fetcher did not exit before timeout.");
        MatcherAssert.assertThat(finishedSplits, Matchers.containsInAnyOrder(KafkaPartitionSplit.toSplitId(normalSplit.getTopicPartition()), KafkaPartitionSplit.toSplitId(emptySplit.getTopicPartition())));
    }
}

Example 24 with KafkaPartitionSplit

use of org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit in project flink by apache.

the class KafkaSourceReaderTest method testKafkaSourceMetrics.

@Test
void testKafkaSourceMetrics() throws Exception {
    final MetricListener metricListener = new MetricListener();
    final String groupId = "testKafkaSourceMetrics";
    final TopicPartition tp0 = new TopicPartition(TOPIC, 0);
    final TopicPartition tp1 = new TopicPartition(TOPIC, 1);
    try (KafkaSourceReader<Integer> reader = (KafkaSourceReader<Integer>) createReader(Boundedness.CONTINUOUS_UNBOUNDED, groupId, metricListener.getMetricGroup())) {
        KafkaPartitionSplit split0 = new KafkaPartitionSplit(tp0, KafkaPartitionSplit.EARLIEST_OFFSET);
        KafkaPartitionSplit split1 = new KafkaPartitionSplit(tp1, KafkaPartitionSplit.EARLIEST_OFFSET);
        reader.addSplits(Arrays.asList(split0, split1));
        TestingReaderOutput<Integer> output = new TestingReaderOutput<>();
        pollUntil(reader, output, () -> output.getEmittedRecords().size() == NUM_RECORDS_PER_SPLIT * 2, String.format("Failed to poll %d records until timeout", NUM_RECORDS_PER_SPLIT * 2));
        // Metric "records-consumed-total" of KafkaConsumer should be NUM_RECORDS_PER_SPLIT
        assertThat(getKafkaConsumerMetric("records-consumed-total", metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT * 2);
        // Current consuming offset should be NUM_RECORD_PER_SPLIT - 1
        assertThat(getCurrentOffsetMetric(tp0, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT - 1);
        assertThat(getCurrentOffsetMetric(tp1, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT - 1);
        // No offset is committed till now
        assertThat(getCommittedOffsetMetric(tp0, metricListener)).isEqualTo(INITIAL_OFFSET);
        assertThat(getCommittedOffsetMetric(tp1, metricListener)).isEqualTo(INITIAL_OFFSET);
        // Trigger offset commit
        final long checkpointId = 15213L;
        reader.snapshotState(checkpointId);
        waitUtil(() -> {
            try {
                reader.notifyCheckpointComplete(checkpointId);
            } catch (Exception e) {
                throw new RuntimeException("Failed to notify checkpoint complete to reader", e);
            }
            return reader.getOffsetsToCommit().isEmpty();
        }, Duration.ofSeconds(60), Duration.ofSeconds(1), String.format("Offsets are not committed successfully. Dangling offsets: %s", reader.getOffsetsToCommit()));
        // Metric "commit-total" of KafkaConsumer should be greater than 0
        // It's hard to know the exactly number of commit because of the retry
        MatcherAssert.assertThat(getKafkaConsumerMetric("commit-total", metricListener), Matchers.greaterThan(0L));
        // Committed offset should be NUM_RECORD_PER_SPLIT
        assertThat(getCommittedOffsetMetric(tp0, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT);
        assertThat(getCommittedOffsetMetric(tp1, metricListener)).isEqualTo(NUM_RECORDS_PER_SPLIT);
        // Number of successful commits should be greater than 0
        final Optional<Counter> commitsSucceeded = metricListener.getCounter(KAFKA_SOURCE_READER_METRIC_GROUP, COMMITS_SUCCEEDED_METRIC_COUNTER);
        assertThat(commitsSucceeded).isPresent();
        MatcherAssert.assertThat(commitsSucceeded.get().getCount(), Matchers.greaterThan(0L));
    }
}

Example 25 with KafkaPartitionSplit

use of org.apache.flink.connector.kafka.source.split.KafkaPartitionSplit in project flink by apache.

the class KafkaEnumeratorTest method testDiscoverPartitionsPeriodically.

@Test(timeout = 30000L)
public void testDiscoverPartitionsPeriodically() throws Throwable {
    try (MockSplitEnumeratorContext<KafkaPartitionSplit> context = new MockSplitEnumeratorContext<>(NUM_SUBTASKS);
        KafkaSourceEnumerator enumerator = createEnumerator(context, ENABLE_PERIODIC_PARTITION_DISCOVERY, INCLUDE_DYNAMIC_TOPIC);
        AdminClient adminClient = KafkaSourceTestEnv.getAdminClient()) {
        startEnumeratorAndRegisterReaders(context, enumerator);
        // invoke partition discovery callable again and there should be no new assignments.
        runPeriodicPartitionDiscovery(context);
        assertEquals("No assignments should be made because there is no partition change", 2, context.getSplitsAssignmentSequence().size());
        // create the dynamic topic.
        adminClient.createTopics(Collections.singleton(new NewTopic(DYNAMIC_TOPIC_NAME, NUM_PARTITIONS_DYNAMIC_TOPIC, (short) 1))).all().get();
        // invoke partition discovery callable again.
        while (true) {
            runPeriodicPartitionDiscovery(context);
            if (context.getSplitsAssignmentSequence().size() < 3) {
                Thread.sleep(10);
            } else {
                break;
            }
        }
        verifyLastReadersAssignments(context, Arrays.asList(READER0, READER1), Collections.singleton(DYNAMIC_TOPIC_NAME), 3);
    } finally {
        try (AdminClient adminClient = KafkaSourceTestEnv.getAdminClient()) {
            adminClient.deleteTopics(Collections.singleton(DYNAMIC_TOPIC_NAME)).all().get();
        } catch (Exception e) {
        // Let it go.
        }
    }
}