Examples with TopicPartition org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition used on opensource projects

Example 11 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarPartitionSplitSerializerTest method serializeAndDeserializePulsarPartitionSplit.

@Test
void serializeAndDeserializePulsarPartitionSplit() throws Exception {
    PulsarPartitionSplit split = new PulsarPartitionSplit(new TopicPartition(randomAlphabetic(10), 10, createFullRange()), StopCursor.defaultStopCursor());
    byte[] bytes = INSTANCE.serialize(split);
    PulsarPartitionSplit split1 = INSTANCE.deserialize(INSTANCE.getVersion(), bytes);
    assertEquals(split, split1);
    assertNotSame(split, split1);
}

Example 12 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarTestCommonUtils method createPartitionSplit.

public static PulsarPartitionSplit createPartitionSplit(String topic, int partitionId, Boundedness boundedness, MessageId latestConsumedId) {
    TopicPartition topicPartition = new TopicPartition(topic, partitionId, TopicRange.createFullRange());
    StopCursor stopCursor = boundedness == Boundedness.BOUNDED ? StopCursor.latest() : StopCursor.never();
    return new PulsarPartitionSplit(topicPartition, stopCursor, latestConsumedId, null);
}

Example 13 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarSubscriberTest method topicPatternSubscriber.

@Test
void topicPatternSubscriber() {
    operator().createTopic(TOPIC1, NUM_PARTITIONS_PER_TOPIC);
    operator().createTopic(TOPIC2, NUM_PARTITIONS_PER_TOPIC);
    operator().createTopic(TOPIC3, NUM_PARTITIONS_PER_TOPIC);
    PulsarSubscriber subscriber = getTopicPatternSubscriber(Pattern.compile("persistent://public/default/topic*?"), AllTopics);
    Set<TopicPartition> topicPartitions = subscriber.getSubscribedTopicPartitions(operator().admin(), new FullRangeGenerator(), NUM_PARALLELISM);
    Set<TopicPartition> expectedPartitions = new HashSet<>();
    for (int i = 0; i < NUM_PARTITIONS_PER_TOPIC; i++) {
        expectedPartitions.add(new TopicPartition(TOPIC1, i, createFullRange()));
        expectedPartitions.add(new TopicPartition(TOPIC3, i, createFullRange()));
    }
    assertEquals(expectedPartitions, topicPartitions);
    operator().deleteTopic(TOPIC1);
    operator().deleteTopic(TOPIC2);
    operator().deleteTopic(TOPIC3);
}

Example 14 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarOrderedPartitionSplitReader method startConsumer.

@Override
protected void startConsumer(PulsarPartitionSplit split, Consumer<byte[]> consumer) {
    MessageId latestConsumedId = split.getLatestConsumedId();
    // Reset the start position for ordered pulsar consumer.
    if (latestConsumedId != null) {
        StartCursor startCursor = StartCursor.fromMessageId(latestConsumedId, false);
        TopicPartition partition = split.getPartition();
        try {
            startCursor.seekPosition(partition.getTopic(), partition.getPartitionId(), consumer);
        } catch (PulsarClientException e) {
            if (sourceConfiguration.getVerifyInitialOffsets() == FAIL_ON_MISMATCH) {
                throw new IllegalArgumentException(e);
            } else {
                // WARN_ON_MISMATCH would just print this warning message.
                // No need to print the stacktrace.
                LOG.warn("Failed to reset cursor to {} on partition {}", latestConsumedId, partition, e);
            }
        }
    }
}

Example 15 with TopicPartition

use of org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition in project flink by apache.

the class PulsarSourceEnumerator method seekStartPosition.

private void seekStartPosition(Set<TopicPartition> partitions) {
    ConsumerBuilder<byte[]> consumerBuilder = consumerBuilder();
    Set<String> seekedTopics = new HashSet<>();
    for (TopicPartition partition : partitions) {
        String topicName = partition.getFullTopicName();
        if (!assignmentState.containsTopic(topicName) && seekedTopics.add(topicName)) {
            try (Consumer<byte[]> consumer = sneakyClient(() -> consumerBuilder.clone().topic(topicName).subscribe())) {
                startCursor.seekPosition(partition.getTopic(), partition.getPartitionId(), consumer);
            } catch (PulsarClientException e) {
                if (sourceConfiguration.getVerifyInitialOffsets() == FAIL_ON_MISMATCH) {
                    throw new IllegalArgumentException(e);
                } else {
                    // WARN_ON_MISMATCH would just print this warning message.
                    // No need to print the stacktrace.
                    LOG.warn("Failed to set initial consuming position for partition {}", partition, e);
                }
            }
        }
    }
}