Examples with SourceConfiguration org.apache.flink.connector.pulsar.source.config.SourceConfiguration used on opensource projects

Example 1 with SourceConfiguration

use of org.apache.flink.connector.pulsar.source.config.SourceConfiguration in project flink by apache.

the class PulsarSourceEnumeratorTest method createEnumerator.

private PulsarSourceEnumerator createEnumerator(SubscriptionType subscriptionType, Set<String> topicsToSubscribe, MockSplitEnumeratorContext<PulsarPartitionSplit> enumContext, boolean enablePeriodicPartitionDiscovery, PulsarSourceEnumState sourceEnumState) {
    // Use a TopicPatternSubscriber so that no exception if a subscribed topic hasn't been
    // created yet.
    String topicRegex = String.join("|", topicsToSubscribe);
    Pattern topicPattern = Pattern.compile(topicRegex);
    PulsarSubscriber subscriber = getTopicPatternSubscriber(topicPattern, RegexSubscriptionMode.AllTopics);
    Configuration configuration = operator().config();
    configuration.set(PULSAR_SUBSCRIPTION_TYPE, subscriptionType);
    if (enablePeriodicPartitionDiscovery) {
        configuration.set(PULSAR_PARTITION_DISCOVERY_INTERVAL_MS, 60L);
    } else {
        configuration.set(PULSAR_PARTITION_DISCOVERY_INTERVAL_MS, -1L);
    }
    SourceConfiguration sourceConfiguration = new SourceConfiguration(configuration);
    SplitsAssignmentState assignmentState = new SplitsAssignmentState(latest(), sourceConfiguration, sourceEnumState);
    return new PulsarSourceEnumerator(subscriber, StartCursor.earliest(), new FullRangeGenerator(), sourceConfiguration, enumContext, assignmentState);
}

Example 2 with SourceConfiguration

use of org.apache.flink.connector.pulsar.source.config.SourceConfiguration in project flink by apache.

the class PulsarPartitionSplitReaderTestBase method sourceConfig.

/**
 * Default source config: max message 1, fetch timeout 1s.
 */
private SourceConfiguration sourceConfig() {
    Configuration config = operator().config();
    config.set(PULSAR_MAX_FETCH_RECORDS, 1);
    config.set(PULSAR_MAX_FETCH_TIME, 1000L);
    config.set(PULSAR_SUBSCRIPTION_NAME, randomAlphabetic(10));
    config.set(PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE, true);
    return new SourceConfiguration(config);
}

Example 3 with SourceConfiguration

use of org.apache.flink.connector.pulsar.source.config.SourceConfiguration in project flink by apache.

the class PulsarSourceBuilder method build.

/**
 * Build the {@link PulsarSource}.
 *
 * @return a PulsarSource with the settings made for this builder.
 */
@SuppressWarnings("java:S3776")
public PulsarSource<OUT> build() {
    // Ensure the topic subscriber for pulsar.
    checkNotNull(subscriber, "No topic names or topic pattern are provided.");
    SubscriptionType subscriptionType = configBuilder.get(PULSAR_SUBSCRIPTION_TYPE);
    if (subscriptionType == SubscriptionType.Key_Shared) {
        if (rangeGenerator == null) {
            LOG.warn("No range generator provided for key_shared subscription," + " we would use the UniformRangeGenerator as the default range generator.");
            this.rangeGenerator = new UniformRangeGenerator();
        }
    } else {
        // Override the range generator.
        this.rangeGenerator = new FullRangeGenerator();
    }
    if (boundedness == null) {
        LOG.warn("No boundedness was set, mark it as a endless stream.");
        this.boundedness = Boundedness.CONTINUOUS_UNBOUNDED;
    }
    if (boundedness == Boundedness.BOUNDED && configBuilder.get(PULSAR_PARTITION_DISCOVERY_INTERVAL_MS) >= 0) {
        LOG.warn("{} property is overridden to -1 because the source is bounded.", PULSAR_PARTITION_DISCOVERY_INTERVAL_MS);
        configBuilder.override(PULSAR_PARTITION_DISCOVERY_INTERVAL_MS, -1L);
    }
    checkNotNull(deserializationSchema, "deserializationSchema should be set.");
    // Enable transaction if the cursor auto commit is disabled for Key_Shared & Shared.
    if (FALSE.equals(configBuilder.get(PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE)) && (subscriptionType == SubscriptionType.Key_Shared || subscriptionType == SubscriptionType.Shared)) {
        LOG.info("Pulsar cursor auto commit is disabled, make sure checkpoint is enabled " + "and your pulsar cluster is support the transaction.");
        configBuilder.override(PULSAR_ENABLE_TRANSACTION, true);
        if (!configBuilder.contains(PULSAR_READ_TRANSACTION_TIMEOUT)) {
            LOG.warn("The default pulsar transaction timeout is 3 hours, " + "make sure it was greater than your checkpoint interval.");
        } else {
            Long timeout = configBuilder.get(PULSAR_READ_TRANSACTION_TIMEOUT);
            LOG.warn("The configured transaction timeout is {} mille seconds, " + "make sure it was greater than your checkpoint interval.", timeout);
        }
    }
    if (!configBuilder.contains(PULSAR_CONSUMER_NAME)) {
        LOG.warn("We recommend set a readable consumer name through setConsumerName(String) in production mode.");
    }
    // Since these implementation could be a lambda, make sure they are serializable.
    checkState(isSerializable(startCursor), "StartCursor isn't serializable");
    checkState(isSerializable(stopCursor), "StopCursor isn't serializable");
    checkState(isSerializable(rangeGenerator), "RangeGenerator isn't serializable");
    // Check builder configuration.
    SourceConfiguration sourceConfiguration = configBuilder.build(SOURCE_CONFIG_VALIDATOR, SourceConfiguration::new);
    return new PulsarSource<>(sourceConfiguration, subscriber, rangeGenerator, startCursor, stopCursor, boundedness, deserializationSchema);
}

Example 4 with SourceConfiguration

use of org.apache.flink.connector.pulsar.source.config.SourceConfiguration in project flink by apache.

the class PulsarSourceReaderTestBase method sourceReader.

private PulsarSourceReaderBase<Integer> sourceReader(boolean autoAcknowledgementEnabled, SubscriptionType subscriptionType) {
    Configuration configuration = operator().config();
    configuration.set(PULSAR_MAX_FETCH_RECORDS, 1);
    configuration.set(PULSAR_MAX_FETCH_TIME, 1000L);
    configuration.set(PULSAR_SUBSCRIPTION_NAME, randomAlphabetic(10));
    configuration.set(PULSAR_SUBSCRIPTION_TYPE, subscriptionType);
    if (autoAcknowledgementEnabled || configuration.get(PULSAR_SUBSCRIPTION_TYPE) == SubscriptionType.Shared) {
        configuration.set(PULSAR_ENABLE_AUTO_ACKNOWLEDGE_MESSAGE, true);
    }
    PulsarDeserializationSchema<Integer> deserializationSchema = pulsarSchema(Schema.INT32);
    SourceReaderContext context = new TestingReaderContext();
    try {
        deserializationSchema.open(new PulsarDeserializationSchemaInitializationContext(context), mock(SourceConfiguration.class));
    } catch (Exception e) {
        fail("Error while opening deserializationSchema");
    }
    SourceConfiguration sourceConfiguration = new SourceConfiguration(configuration);
    return (PulsarSourceReaderBase<Integer>) PulsarSourceReaderFactory.create(context, deserializationSchema, sourceConfiguration);
}

Example 5 with SourceConfiguration

use of org.apache.flink.connector.pulsar.source.config.SourceConfiguration in project flink by apache.

the class SplitsAssignmentStateTest method createConfig.

private SourceConfiguration createConfig(SubscriptionType type) {
    Configuration configuration = new Configuration();
    configuration.set(PULSAR_SUBSCRIPTION_TYPE, type);
    return new SourceConfiguration(configuration);
}