Search in sources :

Example 1 with Exclusive

use of org.apache.pulsar.client.api.SubscriptionType.Exclusive in project flink by apache.

the class PulsarRuntimeOperator method createConsumer.

@SuppressWarnings("unchecked")
private <T> Consumer<T> createConsumer(String topic, Schema<T> schema) throws PulsarClientException {
    TopicName topicName = TopicName.get(topic);
    String name = topicName.getPartitionedTopicName();
    int index = topicName.getPartitionIndex();
    ConcurrentHashMap<Integer, Consumer<?>> topicConsumers = consumers.computeIfAbsent(name, d -> new ConcurrentHashMap<>());
    return (Consumer<T>) topicConsumers.computeIfAbsent(index, i -> {
        try {
            return client().newConsumer(schema).topic(topic).subscriptionName(SUBSCRIPTION_NAME).subscriptionMode(Durable).subscriptionType(Exclusive).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
        } catch (PulsarClientException e) {
            sneakyThrow(e);
            return null;
        }
    });
}
Also used : Strings(org.apache.flink.shaded.guava30.com.google.common.base.Strings) Arrays(java.util.Arrays) Durable(org.apache.pulsar.client.api.SubscriptionMode.Durable) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) PulsarExceptionUtils.sneakyAdmin(org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyAdmin) Producer(org.apache.pulsar.client.api.Producer) EXACTLY_ONCE(org.apache.flink.connector.base.DeliveryGuarantee.EXACTLY_ONCE) NotFoundException(org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException) Random(java.util.Random) Collections.singletonList(java.util.Collections.singletonList) TxnID(org.apache.pulsar.client.api.transaction.TxnID) Collectors.toMap(java.util.stream.Collectors.toMap) Duration(java.time.Duration) Map(java.util.Map) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) ExternalContext(org.apache.flink.connector.testframe.external.ExternalContext) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) PulsarExceptionUtils.sneakyClient(org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyClient) Consumer(org.apache.pulsar.client.api.Consumer) List(java.util.List) Stream(java.util.stream.Stream) Preconditions.checkArgument(org.apache.flink.util.Preconditions.checkArgument) RandomStringUtils.randomAlphanumeric(org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric) Function.identity(java.util.function.Function.identity) PULSAR_ADMIN_URL(org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_URL) PulsarExceptionUtils.sneakyThrow(org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyThrow) PULSAR_ENABLE_TRANSACTION(org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ENABLE_TRANSACTION) DeliveryGuarantee(org.apache.flink.connector.base.DeliveryGuarantee) TopicName(org.apache.pulsar.common.naming.TopicName) TopicRange(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicRange) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Message(org.apache.pulsar.client.api.Message) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) SubscriptionInitialPosition(org.apache.pulsar.client.api.SubscriptionInitialPosition) Supplier(java.util.function.Supplier) TransactionCoordinatorClient(org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient) ArrayList(java.util.ArrayList) Exclusive(org.apache.pulsar.client.api.SubscriptionType.Exclusive) PULSAR_WRITE_TRANSACTION_TIMEOUT(org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_TRANSACTION_TIMEOUT) PulsarClient(org.apache.pulsar.client.api.PulsarClient) TypedMessageBuilder(org.apache.pulsar.client.api.TypedMessageBuilder) PULSAR_SEND_TIMEOUT_MS(org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_SEND_TIMEOUT_MS) TopicNameUtils.topicName(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicName) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Configuration(org.apache.flink.configuration.Configuration) IOException(java.io.IOException) TopicPartition(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition) Schema(org.apache.pulsar.client.api.Schema) ExecutionException(java.util.concurrent.ExecutionException) PulsarConfiguration(org.apache.flink.connector.pulsar.common.config.PulsarConfiguration) Collectors.toList(java.util.stream.Collectors.toList) MessageId(org.apache.pulsar.client.api.MessageId) PULSAR_WRITE_DELIVERY_GUARANTEE(org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_DELIVERY_GUARANTEE) Closeable(java.io.Closeable) TopicNameUtils.topicNameWithPartition(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition) PULSAR_SERVICE_URL(org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_SERVICE_URL) Consumer(org.apache.pulsar.client.api.Consumer) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) TopicName(org.apache.pulsar.common.naming.TopicName)

Example 2 with Exclusive

use of org.apache.pulsar.client.api.SubscriptionType.Exclusive in project flink by splunk.

the class PulsarRuntimeOperator method createConsumer.

@SuppressWarnings("unchecked")
private <T> Consumer<T> createConsumer(String topic, Schema<T> schema) throws PulsarClientException {
    TopicName topicName = TopicName.get(topic);
    String name = topicName.getPartitionedTopicName();
    int index = topicName.getPartitionIndex();
    ConcurrentHashMap<Integer, Consumer<?>> topicConsumers = consumers.computeIfAbsent(name, d -> new ConcurrentHashMap<>());
    return (Consumer<T>) topicConsumers.computeIfAbsent(index, i -> {
        try {
            return client().newConsumer(schema).topic(topic).subscriptionName(SUBSCRIPTION_NAME).subscriptionMode(Durable).subscriptionType(Exclusive).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
        } catch (PulsarClientException e) {
            sneakyThrow(e);
            return null;
        }
    });
}
Also used : Strings(org.apache.flink.shaded.guava30.com.google.common.base.Strings) Arrays(java.util.Arrays) Durable(org.apache.pulsar.client.api.SubscriptionMode.Durable) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) PulsarExceptionUtils.sneakyAdmin(org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyAdmin) Producer(org.apache.pulsar.client.api.Producer) EXACTLY_ONCE(org.apache.flink.connector.base.DeliveryGuarantee.EXACTLY_ONCE) NotFoundException(org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException) Random(java.util.Random) Collections.singletonList(java.util.Collections.singletonList) TxnID(org.apache.pulsar.client.api.transaction.TxnID) Collectors.toMap(java.util.stream.Collectors.toMap) Duration(java.time.Duration) Map(java.util.Map) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) ExternalContext(org.apache.flink.connector.testframe.external.ExternalContext) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) PulsarExceptionUtils.sneakyClient(org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyClient) Consumer(org.apache.pulsar.client.api.Consumer) List(java.util.List) Stream(java.util.stream.Stream) Preconditions.checkArgument(org.apache.flink.util.Preconditions.checkArgument) RandomStringUtils.randomAlphanumeric(org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric) Function.identity(java.util.function.Function.identity) PULSAR_ADMIN_URL(org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_URL) PulsarExceptionUtils.sneakyThrow(org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyThrow) PULSAR_ENABLE_TRANSACTION(org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ENABLE_TRANSACTION) DeliveryGuarantee(org.apache.flink.connector.base.DeliveryGuarantee) TopicName(org.apache.pulsar.common.naming.TopicName) TopicRange(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicRange) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Message(org.apache.pulsar.client.api.Message) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) SubscriptionInitialPosition(org.apache.pulsar.client.api.SubscriptionInitialPosition) Supplier(java.util.function.Supplier) TransactionCoordinatorClient(org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient) ArrayList(java.util.ArrayList) Exclusive(org.apache.pulsar.client.api.SubscriptionType.Exclusive) PULSAR_WRITE_TRANSACTION_TIMEOUT(org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_TRANSACTION_TIMEOUT) PulsarClient(org.apache.pulsar.client.api.PulsarClient) TypedMessageBuilder(org.apache.pulsar.client.api.TypedMessageBuilder) PULSAR_SEND_TIMEOUT_MS(org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_SEND_TIMEOUT_MS) TopicNameUtils.topicName(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicName) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Configuration(org.apache.flink.configuration.Configuration) IOException(java.io.IOException) TopicPartition(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition) Schema(org.apache.pulsar.client.api.Schema) ExecutionException(java.util.concurrent.ExecutionException) PulsarConfiguration(org.apache.flink.connector.pulsar.common.config.PulsarConfiguration) Collectors.toList(java.util.stream.Collectors.toList) MessageId(org.apache.pulsar.client.api.MessageId) PULSAR_WRITE_DELIVERY_GUARANTEE(org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_DELIVERY_GUARANTEE) Closeable(java.io.Closeable) TopicNameUtils.topicNameWithPartition(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition) PULSAR_SERVICE_URL(org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_SERVICE_URL) Consumer(org.apache.pulsar.client.api.Consumer) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) TopicName(org.apache.pulsar.common.naming.TopicName)

Example 3 with Exclusive

use of org.apache.pulsar.client.api.SubscriptionType.Exclusive in project flink-mirror by flink-ci.

the class PulsarRuntimeOperator method createConsumer.

@SuppressWarnings("unchecked")
private <T> Consumer<T> createConsumer(String topic, Schema<T> schema) throws PulsarClientException {
    TopicName topicName = TopicName.get(topic);
    String name = topicName.getPartitionedTopicName();
    int index = topicName.getPartitionIndex();
    ConcurrentHashMap<Integer, Consumer<?>> topicConsumers = consumers.computeIfAbsent(name, d -> new ConcurrentHashMap<>());
    return (Consumer<T>) topicConsumers.computeIfAbsent(index, i -> {
        try {
            return client().newConsumer(schema).topic(topic).subscriptionName(SUBSCRIPTION_NAME).subscriptionMode(Durable).subscriptionType(Exclusive).subscriptionInitialPosition(SubscriptionInitialPosition.Earliest).subscribe();
        } catch (PulsarClientException e) {
            sneakyThrow(e);
            return null;
        }
    });
}
Also used : Strings(org.apache.flink.shaded.guava30.com.google.common.base.Strings) Arrays(java.util.Arrays) Durable(org.apache.pulsar.client.api.SubscriptionMode.Durable) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) PulsarExceptionUtils.sneakyAdmin(org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyAdmin) Producer(org.apache.pulsar.client.api.Producer) EXACTLY_ONCE(org.apache.flink.connector.base.DeliveryGuarantee.EXACTLY_ONCE) NotFoundException(org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException) Random(java.util.Random) Collections.singletonList(java.util.Collections.singletonList) TxnID(org.apache.pulsar.client.api.transaction.TxnID) Collectors.toMap(java.util.stream.Collectors.toMap) Duration(java.time.Duration) Map(java.util.Map) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) ExternalContext(org.apache.flink.connector.testframe.external.ExternalContext) Collections.emptyList(java.util.Collections.emptyList) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) MILLISECONDS(java.util.concurrent.TimeUnit.MILLISECONDS) PulsarExceptionUtils.sneakyClient(org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyClient) Consumer(org.apache.pulsar.client.api.Consumer) List(java.util.List) Stream(java.util.stream.Stream) Preconditions.checkArgument(org.apache.flink.util.Preconditions.checkArgument) RandomStringUtils.randomAlphanumeric(org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric) Function.identity(java.util.function.Function.identity) PULSAR_ADMIN_URL(org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ADMIN_URL) PulsarExceptionUtils.sneakyThrow(org.apache.flink.connector.pulsar.common.utils.PulsarExceptionUtils.sneakyThrow) PULSAR_ENABLE_TRANSACTION(org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_ENABLE_TRANSACTION) DeliveryGuarantee(org.apache.flink.connector.base.DeliveryGuarantee) TopicName(org.apache.pulsar.common.naming.TopicName) TopicRange(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicRange) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) Message(org.apache.pulsar.client.api.Message) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) SubscriptionInitialPosition(org.apache.pulsar.client.api.SubscriptionInitialPosition) Supplier(java.util.function.Supplier) TransactionCoordinatorClient(org.apache.pulsar.client.api.transaction.TransactionCoordinatorClient) ArrayList(java.util.ArrayList) Exclusive(org.apache.pulsar.client.api.SubscriptionType.Exclusive) PULSAR_WRITE_TRANSACTION_TIMEOUT(org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_TRANSACTION_TIMEOUT) PulsarClient(org.apache.pulsar.client.api.PulsarClient) TypedMessageBuilder(org.apache.pulsar.client.api.TypedMessageBuilder) PULSAR_SEND_TIMEOUT_MS(org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_SEND_TIMEOUT_MS) TopicNameUtils.topicName(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicName) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) Configuration(org.apache.flink.configuration.Configuration) IOException(java.io.IOException) TopicPartition(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicPartition) Schema(org.apache.pulsar.client.api.Schema) ExecutionException(java.util.concurrent.ExecutionException) PulsarConfiguration(org.apache.flink.connector.pulsar.common.config.PulsarConfiguration) Collectors.toList(java.util.stream.Collectors.toList) MessageId(org.apache.pulsar.client.api.MessageId) PULSAR_WRITE_DELIVERY_GUARANTEE(org.apache.flink.connector.pulsar.sink.PulsarSinkOptions.PULSAR_WRITE_DELIVERY_GUARANTEE) Closeable(java.io.Closeable) TopicNameUtils.topicNameWithPartition(org.apache.flink.connector.pulsar.source.enumerator.topic.TopicNameUtils.topicNameWithPartition) PULSAR_SERVICE_URL(org.apache.flink.connector.pulsar.common.config.PulsarOptions.PULSAR_SERVICE_URL) Consumer(org.apache.pulsar.client.api.Consumer) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) TopicName(org.apache.pulsar.common.naming.TopicName)

Aggregations

Closeable (java.io.Closeable)3 IOException (java.io.IOException)3 Duration (java.time.Duration)3 ArrayList (java.util.ArrayList)3 Arrays (java.util.Arrays)3 Collection (java.util.Collection)3 Collections.emptyList (java.util.Collections.emptyList)3 Collections.singletonList (java.util.Collections.singletonList)3 List (java.util.List)3 Map (java.util.Map)3 Random (java.util.Random)3 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)3 ExecutionException (java.util.concurrent.ExecutionException)3 MILLISECONDS (java.util.concurrent.TimeUnit.MILLISECONDS)3 Function.identity (java.util.function.Function.identity)3 Supplier (java.util.function.Supplier)3 Collectors.toList (java.util.stream.Collectors.toList)3 Collectors.toMap (java.util.stream.Collectors.toMap)3 Stream (java.util.stream.Stream)3 RandomStringUtils.randomAlphanumeric (org.apache.commons.lang3.RandomStringUtils.randomAlphanumeric)3