Examples with FullRangeGenerator org.apache.flink.connector.pulsar.source.enumerator.topic.range.FullRangeGenerator used on opensource projects

Example 1 with FullRangeGenerator

use of org.apache.flink.connector.pulsar.source.enumerator.topic.range.FullRangeGenerator in project flink by apache.

the class PulsarSourceEnumeratorTest method createEnumerator.

private PulsarSourceEnumerator createEnumerator(SubscriptionType subscriptionType, Set<String> topicsToSubscribe, MockSplitEnumeratorContext<PulsarPartitionSplit> enumContext, boolean enablePeriodicPartitionDiscovery, PulsarSourceEnumState sourceEnumState) {
    // Use a TopicPatternSubscriber so that no exception if a subscribed topic hasn't been
    // created yet.
    String topicRegex = String.join("|", topicsToSubscribe);
    Pattern topicPattern = Pattern.compile(topicRegex);
    PulsarSubscriber subscriber = getTopicPatternSubscriber(topicPattern, RegexSubscriptionMode.AllTopics);
    Configuration configuration = operator().config();
    configuration.set(PULSAR_SUBSCRIPTION_TYPE, subscriptionType);
    if (enablePeriodicPartitionDiscovery) {
        configuration.set(PULSAR_PARTITION_DISCOVERY_INTERVAL_MS, 60L);
    } else {
        configuration.set(PULSAR_PARTITION_DISCOVERY_INTERVAL_MS, -1L);
    }
    SourceConfiguration sourceConfiguration = new SourceConfiguration(configuration);
    SplitsAssignmentState assignmentState = new SplitsAssignmentState(latest(), sourceConfiguration, sourceEnumState);
    return new PulsarSourceEnumerator(subscriber, StartCursor.earliest(), new FullRangeGenerator(), sourceConfiguration, enumContext, assignmentState);
}

Example 2 with FullRangeGenerator

use of org.apache.flink.connector.pulsar.source.enumerator.topic.range.FullRangeGenerator in project flink by apache.

the class PulsarSubscriberTest method topicListSubscriber.

@Test
void topicListSubscriber() {
    operator().createTopic(TOPIC1, NUM_PARTITIONS_PER_TOPIC);
    operator().createTopic(TOPIC2, NUM_PARTITIONS_PER_TOPIC);
    PulsarSubscriber subscriber = getTopicListSubscriber(Arrays.asList(TOPIC1, TOPIC2));
    Set<TopicPartition> topicPartitions = subscriber.getSubscribedTopicPartitions(operator().admin(), new FullRangeGenerator(), NUM_PARALLELISM);
    Set<TopicPartition> expectedPartitions = new HashSet<>();
    for (int i = 0; i < NUM_PARTITIONS_PER_TOPIC; i++) {
        expectedPartitions.add(new TopicPartition(TOPIC1, i, createFullRange()));
        expectedPartitions.add(new TopicPartition(TOPIC2, i, createFullRange()));
    }
    assertEquals(expectedPartitions, topicPartitions);
    operator().deleteTopic(TOPIC1);
    operator().deleteTopic(TOPIC2);
}

Example 3 with FullRangeGenerator

use of org.apache.flink.connector.pulsar.source.enumerator.topic.range.FullRangeGenerator in project flink by apache.

the class PulsarSourceBuilder method build.

/**
 * Build the {@link PulsarSource}.
 *
 * @return a PulsarSource with the settings made for this builder.
 */
@SuppressWarnings("java:S3776")
public PulsarSource<OUT> build() {
    // Ensure the topic subscriber for pulsar.
    checkNotNull(subscriber, "No topic names or topic pattern are provided.");
    SubscriptionType subscriptionType = configBuilder.get(PULSAR_SUBSCRIPTION_TYPE);
    if (subscriptionType == SubscriptionType.Key_Shared) {
        if (rangeGenerator == null) {
            LOG.warn("No range generator provided for key_shared subscription," + " we would use the UniformRangeGenerator as the default range generator.");
            this.rangeGenerator = new UniformRangeGenerator();
        }
    } else {
        // Override the range generator.
        this.rangeGenerator = new FullRangeGenerator();
    }
    if (boundedness == null) {
        LOG.warn("No boundedness was set, mark it as a endless stream.");
        this.boundedness = Boundedness.CONTINUOUS_UNBOUNDED;
    }
    if (boundedness == Boundedness.BOUNDED && configBuilder.get(PULSAR_PARTITION_DISCOVERY_INTERVAL_MS) >= 0) {
        LOG.warn("{} property is overridden to -1 because the source is bounded.", PULSAR_PARTITION_DISCOVERY_INTERVAL_MS);
        configBuilder.override(PULSAR_PARTITION_DISCOVERY_INTERVAL_MS, -1L);
    }
    checkNotNull(deserializationSchema, "deserializationSchema should be set.");
    // Enable transaction if the cursor auto commit is disabled for Key_Shared & Shared.
    if (FALSE.equals(configBuilder.get(PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE)) && (subscriptionType == SubscriptionType.Key_Shared || subscriptionType == SubscriptionType.Shared)) {
        LOG.info("Pulsar cursor auto commit is disabled, make sure checkpoint is enabled " + "and your pulsar cluster is support the transaction.");
        configBuilder.override(PULSAR_ENABLE_TRANSACTION, true);
        if (!configBuilder.contains(PULSAR_READ_TRANSACTION_TIMEOUT)) {
            LOG.warn("The default pulsar transaction timeout is 3 hours, " + "make sure it was greater than your checkpoint interval.");
        } else {
            Long timeout = configBuilder.get(PULSAR_READ_TRANSACTION_TIMEOUT);
            LOG.warn("The configured transaction timeout is {} mille seconds, " + "make sure it was greater than your checkpoint interval.", timeout);
        }
    }
    if (!configBuilder.contains(PULSAR_CONSUMER_NAME)) {
        LOG.warn("We recommend set a readable consumer name through setConsumerName(String) in production mode.");
    }
    // Since these implementation could be a lambda, make sure they are serializable.
    checkState(isSerializable(startCursor), "StartCursor isn't serializable");
    checkState(isSerializable(stopCursor), "StopCursor isn't serializable");
    checkState(isSerializable(rangeGenerator), "RangeGenerator isn't serializable");
    // Check builder configuration.
    SourceConfiguration sourceConfiguration = configBuilder.build(SOURCE_CONFIG_VALIDATOR, SourceConfiguration::new);
    return new PulsarSource<>(sourceConfiguration, subscriber, rangeGenerator, startCursor, stopCursor, boundedness, deserializationSchema);
}

Example 4 with FullRangeGenerator

use of org.apache.flink.connector.pulsar.source.enumerator.topic.range.FullRangeGenerator in project flink by apache.

the class PulsarSubscriberTest method topicPatternSubscriber.

@Test
void topicPatternSubscriber() {
    operator().createTopic(TOPIC1, NUM_PARTITIONS_PER_TOPIC);
    operator().createTopic(TOPIC2, NUM_PARTITIONS_PER_TOPIC);
    operator().createTopic(TOPIC3, NUM_PARTITIONS_PER_TOPIC);
    PulsarSubscriber subscriber = getTopicPatternSubscriber(Pattern.compile("persistent://public/default/topic*?"), AllTopics);
    Set<TopicPartition> topicPartitions = subscriber.getSubscribedTopicPartitions(operator().admin(), new FullRangeGenerator(), NUM_PARALLELISM);
    Set<TopicPartition> expectedPartitions = new HashSet<>();
    for (int i = 0; i < NUM_PARTITIONS_PER_TOPIC; i++) {
        expectedPartitions.add(new TopicPartition(TOPIC1, i, createFullRange()));
        expectedPartitions.add(new TopicPartition(TOPIC3, i, createFullRange()));
    }
    assertEquals(expectedPartitions, topicPartitions);
    operator().deleteTopic(TOPIC1);
    operator().deleteTopic(TOPIC2);
    operator().deleteTopic(TOPIC3);
}