Examples with KeySharedPolicy - org.apache.pulsar.client.api.KeySharedPolicy

Example 1 with KeySharedPolicy

use of org.apache.pulsar.client.api.KeySharedPolicy in project pulsar by yahoo.

the class ConsumerBuilderImpl method subscribeAsync.

@Override
public CompletableFuture<Consumer<T>> subscribeAsync() {
    if (conf.getTopicNames().isEmpty() && conf.getTopicsPattern() == null) {
        return FutureUtil.failedFuture(new InvalidConfigurationException("Topic name must be set on the consumer builder"));
    }
    if (StringUtils.isBlank(conf.getSubscriptionName())) {
        return FutureUtil.failedFuture(new InvalidConfigurationException("Subscription name must be set on the consumer builder"));
    }
    if (conf.getKeySharedPolicy() != null && conf.getSubscriptionType() != SubscriptionType.Key_Shared) {
        return FutureUtil.failedFuture(new InvalidConfigurationException("KeySharedPolicy must set with KeyShared subscription"));
    }
    CompletableFuture<Void> applyDLQConfig;
    if (conf.isRetryEnable() && conf.getTopicNames().size() > 0) {
        TopicName topicFirst = TopicName.get(conf.getTopicNames().iterator().next());
        // Issue 9327: do compatibility check in case of the default retry and dead letter topic name changed
        String oldRetryLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX;
        String oldDeadLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX;
        DeadLetterPolicy deadLetterPolicy = conf.getDeadLetterPolicy();
        if (deadLetterPolicy == null || StringUtils.isBlank(deadLetterPolicy.getRetryLetterTopic()) || StringUtils.isBlank(deadLetterPolicy.getDeadLetterTopic())) {
            CompletableFuture<PartitionedTopicMetadata> retryLetterTopicMetadata = client.getPartitionedTopicMetadata(oldRetryLetterTopic);
            CompletableFuture<PartitionedTopicMetadata> deadLetterTopicMetadata = client.getPartitionedTopicMetadata(oldDeadLetterTopic);
            applyDLQConfig = CompletableFuture.allOf(retryLetterTopicMetadata, deadLetterTopicMetadata).thenAccept(__ -> {
                String retryLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX;
                String deadLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX;
                if (retryLetterTopicMetadata.join().partitions > 0) {
                    retryLetterTopic = oldRetryLetterTopic;
                }
                if (deadLetterTopicMetadata.join().partitions > 0) {
                    deadLetterTopic = oldDeadLetterTopic;
                }
                if (deadLetterPolicy == null) {
                    conf.setDeadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(RetryMessageUtil.MAX_RECONSUMETIMES).retryLetterTopic(retryLetterTopic).deadLetterTopic(deadLetterTopic).build());
                } else {
                    if (StringUtils.isBlank(deadLetterPolicy.getRetryLetterTopic())) {
                        conf.getDeadLetterPolicy().setRetryLetterTopic(retryLetterTopic);
                    }
                    if (StringUtils.isBlank(deadLetterPolicy.getDeadLetterTopic())) {
                        conf.getDeadLetterPolicy().setDeadLetterTopic(deadLetterTopic);
                    }
                }
                conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic());
            });
        } else {
            conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic());
            applyDLQConfig = CompletableFuture.completedFuture(null);
        }
    } else {
        applyDLQConfig = CompletableFuture.completedFuture(null);
    }
    return applyDLQConfig.thenCompose(__ -> {
        if (interceptorList == null || interceptorList.size() == 0) {
            return client.subscribeAsync(conf, schema, null);
        } else {
            return client.subscribeAsync(conf, schema, new ConsumerInterceptors<>(interceptorList));
        }
    });
}

Also used : Arrays(java.util.Arrays) ConsumerCryptoFailureAction(org.apache.pulsar.client.api.ConsumerCryptoFailureAction) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) ConsumerConfigurationData(org.apache.pulsar.client.impl.conf.ConsumerConfigurationData) RegexSubscriptionMode(org.apache.pulsar.client.api.RegexSubscriptionMode) SubscriptionMode(org.apache.pulsar.client.api.SubscriptionMode) CompletableFuture(java.util.concurrent.CompletableFuture) ConsumerBuilder(org.apache.pulsar.client.api.ConsumerBuilder) MessageCrypto(org.apache.pulsar.client.api.MessageCrypto) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) SubscriptionInitialPosition(org.apache.pulsar.client.api.SubscriptionInitialPosition) StringUtils(org.apache.commons.lang3.StringUtils) ConsumerEventListener(org.apache.pulsar.client.api.ConsumerEventListener) ArrayList(java.util.ArrayList) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) AccessLevel(lombok.AccessLevel) BatchReceivePolicy(org.apache.pulsar.client.api.BatchReceivePolicy) RedeliveryBackoff(org.apache.pulsar.client.api.RedeliveryBackoff) Map(java.util.Map) KeySharedPolicy(org.apache.pulsar.client.api.KeySharedPolicy) RetryMessageUtil(org.apache.pulsar.client.util.RetryMessageUtil) InvalidConfigurationException(org.apache.pulsar.client.api.PulsarClientException.InvalidConfigurationException) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) NonNull(lombok.NonNull) MessageListener(org.apache.pulsar.client.api.MessageListener) DeadLetterPolicy(org.apache.pulsar.client.api.DeadLetterPolicy) SubscriptionType(org.apache.pulsar.client.api.SubscriptionType) Collectors(java.util.stream.Collectors) Schema(org.apache.pulsar.client.api.Schema) ConfigurationDataUtils(org.apache.pulsar.client.impl.conf.ConfigurationDataUtils) TimeUnit(java.util.concurrent.TimeUnit) Consumer(org.apache.pulsar.client.api.Consumer) CryptoKeyReader(org.apache.pulsar.client.api.CryptoKeyReader) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) ConsumerInterceptor(org.apache.pulsar.client.api.ConsumerInterceptor) Pattern(java.util.regex.Pattern) Collections(java.util.Collections) MessagePayloadProcessor(org.apache.pulsar.client.api.MessagePayloadProcessor) DeadLetterPolicy(org.apache.pulsar.client.api.DeadLetterPolicy) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) InvalidConfigurationException(org.apache.pulsar.client.api.PulsarClientException.InvalidConfigurationException) TopicName(org.apache.pulsar.common.naming.TopicName)

Example 2 with KeySharedPolicy

use of org.apache.pulsar.client.api.KeySharedPolicy in project flink by apache.

the class PulsarPartitionSplitReaderBase method createPulsarConsumer.

/**
 * Create a specified {@link Consumer} by the given topic partition.
 */
protected Consumer<byte[]> createPulsarConsumer(TopicPartition partition) {
    ConsumerBuilder<byte[]> consumerBuilder = createConsumerBuilder(pulsarClient, Schema.BYTES, sourceConfiguration);
    consumerBuilder.topic(partition.getFullTopicName());
    // Add KeySharedPolicy for Key_Shared subscription.
    if (sourceConfiguration.getSubscriptionType() == SubscriptionType.Key_Shared) {
        KeySharedPolicy policy = KeySharedPolicy.stickyHashRange().ranges(partition.getPulsarRange());
        consumerBuilder.keySharedPolicy(policy);
    }
    // Create the consumer configuration by using common utils.
    return sneakyClient(consumerBuilder::subscribe);
}

Also used : KeySharedPolicy(org.apache.pulsar.client.api.KeySharedPolicy)

Example 3 with KeySharedPolicy

use of org.apache.pulsar.client.api.KeySharedPolicy in project incubator-pulsar by apache.

the class ConsumerBuilderImpl method subscribeAsync.

@Override
public CompletableFuture<Consumer<T>> subscribeAsync() {
    if (conf.getTopicNames().isEmpty() && conf.getTopicsPattern() == null) {
        return FutureUtil.failedFuture(new InvalidConfigurationException("Topic name must be set on the consumer builder"));
    }
    if (StringUtils.isBlank(conf.getSubscriptionName())) {
        return FutureUtil.failedFuture(new InvalidConfigurationException("Subscription name must be set on the consumer builder"));
    }
    if (conf.getKeySharedPolicy() != null && conf.getSubscriptionType() != SubscriptionType.Key_Shared) {
        return FutureUtil.failedFuture(new InvalidConfigurationException("KeySharedPolicy must set with KeyShared subscription"));
    }
    CompletableFuture<Void> applyDLQConfig;
    if (conf.isRetryEnable() && conf.getTopicNames().size() > 0) {
        TopicName topicFirst = TopicName.get(conf.getTopicNames().iterator().next());
        // Issue 9327: do compatibility check in case of the default retry and dead letter topic name changed
        String oldRetryLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX;
        String oldDeadLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX;
        DeadLetterPolicy deadLetterPolicy = conf.getDeadLetterPolicy();
        if (deadLetterPolicy == null || StringUtils.isBlank(deadLetterPolicy.getRetryLetterTopic()) || StringUtils.isBlank(deadLetterPolicy.getDeadLetterTopic())) {
            CompletableFuture<PartitionedTopicMetadata> retryLetterTopicMetadata = client.getPartitionedTopicMetadata(oldRetryLetterTopic);
            CompletableFuture<PartitionedTopicMetadata> deadLetterTopicMetadata = client.getPartitionedTopicMetadata(oldDeadLetterTopic);
            applyDLQConfig = CompletableFuture.allOf(retryLetterTopicMetadata, deadLetterTopicMetadata).thenAccept(__ -> {
                String retryLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX;
                String deadLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX;
                if (retryLetterTopicMetadata.join().partitions > 0) {
                    retryLetterTopic = oldRetryLetterTopic;
                }
                if (deadLetterTopicMetadata.join().partitions > 0) {
                    deadLetterTopic = oldDeadLetterTopic;
                }
                if (deadLetterPolicy == null) {
                    conf.setDeadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(RetryMessageUtil.MAX_RECONSUMETIMES).retryLetterTopic(retryLetterTopic).deadLetterTopic(deadLetterTopic).build());
                } else {
                    if (StringUtils.isBlank(deadLetterPolicy.getRetryLetterTopic())) {
                        conf.getDeadLetterPolicy().setRetryLetterTopic(retryLetterTopic);
                    }
                    if (StringUtils.isBlank(deadLetterPolicy.getDeadLetterTopic())) {
                        conf.getDeadLetterPolicy().setDeadLetterTopic(deadLetterTopic);
                    }
                }
                conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic());
            });
        } else {
            conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic());
            applyDLQConfig = CompletableFuture.completedFuture(null);
        }
    } else {
        applyDLQConfig = CompletableFuture.completedFuture(null);
    }
    return applyDLQConfig.thenCompose(__ -> {
        if (interceptorList == null || interceptorList.size() == 0) {
            return client.subscribeAsync(conf, schema, null);
        } else {
            return client.subscribeAsync(conf, schema, new ConsumerInterceptors<>(interceptorList));
        }
    });
}

Example 4 with KeySharedPolicy

use of org.apache.pulsar.client.api.KeySharedPolicy in project flink by splunk.

the class PulsarPartitionSplitReaderBase method createPulsarConsumer.

/**
 * Create a specified {@link Consumer} by the given topic partition.
 */
protected Consumer<byte[]> createPulsarConsumer(TopicPartition partition) {
    ConsumerBuilder<byte[]> consumerBuilder = createConsumerBuilder(pulsarClient, Schema.BYTES, sourceConfiguration);
    consumerBuilder.topic(partition.getFullTopicName());
    // Add KeySharedPolicy for Key_Shared subscription.
    if (sourceConfiguration.getSubscriptionType() == SubscriptionType.Key_Shared) {
        KeySharedPolicy policy = KeySharedPolicy.stickyHashRange().ranges(partition.getPulsarRange());
        consumerBuilder.keySharedPolicy(policy);
    }
    // Create the consumer configuration by using common utils.
    return sneakyClient(consumerBuilder::subscribe);
}

Also used : KeySharedPolicy(org.apache.pulsar.client.api.KeySharedPolicy)

Example 5 with KeySharedPolicy

use of org.apache.pulsar.client.api.KeySharedPolicy in project pulsar by apache.

the class ConsumerBuilderImpl method subscribeAsync.

@Override
public CompletableFuture<Consumer<T>> subscribeAsync() {
    if (conf.getTopicNames().isEmpty() && conf.getTopicsPattern() == null) {
        return FutureUtil.failedFuture(new InvalidConfigurationException("Topic name must be set on the consumer builder"));
    }
    if (StringUtils.isBlank(conf.getSubscriptionName())) {
        return FutureUtil.failedFuture(new InvalidConfigurationException("Subscription name must be set on the consumer builder"));
    }
    if (conf.getKeySharedPolicy() != null && conf.getSubscriptionType() != SubscriptionType.Key_Shared) {
        return FutureUtil.failedFuture(new InvalidConfigurationException("KeySharedPolicy must set with KeyShared subscription"));
    }
    CompletableFuture<Void> applyDLQConfig;
    if (conf.isRetryEnable() && conf.getTopicNames().size() > 0) {
        TopicName topicFirst = TopicName.get(conf.getTopicNames().iterator().next());
        // Issue 9327: do compatibility check in case of the default retry and dead letter topic name changed
        String oldRetryLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX;
        String oldDeadLetterTopic = topicFirst.getNamespace() + "/" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX;
        DeadLetterPolicy deadLetterPolicy = conf.getDeadLetterPolicy();
        if (deadLetterPolicy == null || StringUtils.isBlank(deadLetterPolicy.getRetryLetterTopic()) || StringUtils.isBlank(deadLetterPolicy.getDeadLetterTopic())) {
            CompletableFuture<PartitionedTopicMetadata> retryLetterTopicMetadata = client.getPartitionedTopicMetadata(oldRetryLetterTopic);
            CompletableFuture<PartitionedTopicMetadata> deadLetterTopicMetadata = client.getPartitionedTopicMetadata(oldDeadLetterTopic);
            applyDLQConfig = CompletableFuture.allOf(retryLetterTopicMetadata, deadLetterTopicMetadata).thenAccept(__ -> {
                String retryLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + RetryMessageUtil.RETRY_GROUP_TOPIC_SUFFIX;
                String deadLetterTopic = topicFirst + "-" + conf.getSubscriptionName() + RetryMessageUtil.DLQ_GROUP_TOPIC_SUFFIX;
                if (retryLetterTopicMetadata.join().partitions > 0) {
                    retryLetterTopic = oldRetryLetterTopic;
                }
                if (deadLetterTopicMetadata.join().partitions > 0) {
                    deadLetterTopic = oldDeadLetterTopic;
                }
                if (deadLetterPolicy == null) {
                    conf.setDeadLetterPolicy(DeadLetterPolicy.builder().maxRedeliverCount(RetryMessageUtil.MAX_RECONSUMETIMES).retryLetterTopic(retryLetterTopic).deadLetterTopic(deadLetterTopic).build());
                } else {
                    if (StringUtils.isBlank(deadLetterPolicy.getRetryLetterTopic())) {
                        conf.getDeadLetterPolicy().setRetryLetterTopic(retryLetterTopic);
                    }
                    if (StringUtils.isBlank(deadLetterPolicy.getDeadLetterTopic())) {
                        conf.getDeadLetterPolicy().setDeadLetterTopic(deadLetterTopic);
                    }
                }
                conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic());
            });
        } else {
            conf.getTopicNames().add(conf.getDeadLetterPolicy().getRetryLetterTopic());
            applyDLQConfig = CompletableFuture.completedFuture(null);
        }
    } else {
        applyDLQConfig = CompletableFuture.completedFuture(null);
    }
    return applyDLQConfig.thenCompose(__ -> {
        if (interceptorList == null || interceptorList.size() == 0) {
            return client.subscribeAsync(conf, schema, null);
        } else {
            return client.subscribeAsync(conf, schema, new ConsumerInterceptors<>(interceptorList));
        }
    });
}

Aggregations

KeySharedPolicy (org.apache.pulsar.client.api.KeySharedPolicy)6 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)3 ArrayList (java.util.ArrayList)3 Arrays (java.util.Arrays)3 Collections (java.util.Collections)3 List (java.util.List)3 Map (java.util.Map)3 CompletableFuture (java.util.concurrent.CompletableFuture)3 TimeUnit (java.util.concurrent.TimeUnit)3 Pattern (java.util.regex.Pattern)3 Collectors (java.util.stream.Collectors)3 AccessLevel (lombok.AccessLevel)3 Getter (lombok.Getter)3 NonNull (lombok.NonNull)3 StringUtils (org.apache.commons.lang3.StringUtils)3 BatchReceivePolicy (org.apache.pulsar.client.api.BatchReceivePolicy)3 Consumer (org.apache.pulsar.client.api.Consumer)3 ConsumerBuilder (org.apache.pulsar.client.api.ConsumerBuilder)3 ConsumerCryptoFailureAction (org.apache.pulsar.client.api.ConsumerCryptoFailureAction)3 ConsumerEventListener (org.apache.pulsar.client.api.ConsumerEventListener)3