Search in sources :

Example 1 with SubType

use of org.apache.pulsar.common.api.proto.CommandSubscribe.SubType in project pulsar by apache.

the class Consumer method individualAckWithTransaction.

// this method is for individual ack carry the transaction
private CompletableFuture<Void> individualAckWithTransaction(CommandAck ack) {
    // Individual ack
    List<MutablePair<PositionImpl, Integer>> positionsAcked = new ArrayList<>();
    if (!isTransactionEnabled()) {
        return FutureUtil.failedFuture(new BrokerServiceException.NotAllowedException("Server don't support transaction ack!"));
    }
    for (int i = 0; i < ack.getMessageIdsCount(); i++) {
        MessageIdData msgId = ack.getMessageIdAt(i);
        PositionImpl position;
        long ackedCount = 0;
        long batchSize = getBatchSize(msgId);
        Consumer ackOwnerConsumer = getAckOwnerConsumer(msgId.getLedgerId(), msgId.getEntryId());
        if (msgId.getAckSetsCount() > 0) {
            long[] ackSets = new long[msgId.getAckSetsCount()];
            for (int j = 0; j < msgId.getAckSetsCount(); j++) {
                ackSets[j] = msgId.getAckSetAt(j);
            }
            position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId(), ackSets);
            ackedCount = getAckedCountForTransactionAck(batchSize, ackSets);
        } else {
            position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId());
            ackedCount = batchSize;
        }
        positionsAcked.add(new MutablePair<>(position, (int) batchSize));
        addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount);
        checkCanRemovePendingAcksAndHandle(position, msgId);
        checkAckValidationError(ack, position);
    }
    CompletableFuture<Void> completableFuture = transactionIndividualAcknowledge(ack.getTxnidMostBits(), ack.getTxnidLeastBits(), positionsAcked);
    if (Subscription.isIndividualAckMode(subType)) {
        completableFuture.whenComplete((v, e) -> positionsAcked.forEach(positionLongMutablePair -> {
            if (positionLongMutablePair.getLeft().getAckSet() != null) {
                if (((PersistentSubscription) subscription).checkIsCanDeleteConsumerPendingAck(positionLongMutablePair.left)) {
                    removePendingAcks(positionLongMutablePair.left);
                }
            }
        }));
    }
    return completableFuture;
}
Also used : LongAdder(java.util.concurrent.atomic.LongAdder) AtomicIntegerFieldUpdater(java.util.concurrent.atomic.AtomicIntegerFieldUpdater) Setter(lombok.Setter) MutableInt(org.apache.commons.lang3.mutable.MutableInt) TopicName(org.apache.pulsar.common.naming.TopicName) LongPair(org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair) Getter(lombok.Getter) AckType(org.apache.pulsar.common.api.proto.CommandAck.AckType) MessageIdData(org.apache.pulsar.common.api.proto.MessageIdData) Entry(org.apache.bookkeeper.mledger.Entry) LoggerFactory(org.slf4j.LoggerFactory) CompletableFuture(java.util.concurrent.CompletableFuture) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) ArrayList(java.util.ArrayList) Commands(org.apache.pulsar.common.protocol.Commands) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) TxnID(org.apache.pulsar.client.api.transaction.TxnID) Lists(com.google.common.collect.Lists) KeyLongValue(org.apache.pulsar.common.api.proto.KeyLongValue) DateFormatter(org.apache.pulsar.common.util.DateFormatter) MutablePair(org.apache.commons.lang3.tuple.MutablePair) Map(java.util.Map) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) CommandAck(org.apache.pulsar.common.api.proto.CommandAck) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) Logger(org.slf4j.Logger) Promise(io.netty.util.concurrent.Promise) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) TransactionConflictException(org.apache.pulsar.transaction.common.exception.TransactionConflictException) BitSetRecyclable(org.apache.pulsar.common.util.collections.BitSetRecyclable) MoreObjects(com.google.common.base.MoreObjects) Position(org.apache.bookkeeper.mledger.Position) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) FutureUtil(org.apache.pulsar.common.util.FutureUtil) MessageId(org.apache.pulsar.client.api.MessageId) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription) ConcurrentLongLongPairHashMap(org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap) BitSet(java.util.BitSet) Future(io.netty.util.concurrent.Future) Collections(java.util.Collections) Rate(org.apache.pulsar.common.stats.Rate) MessageIdData(org.apache.pulsar.common.api.proto.MessageIdData) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) ArrayList(java.util.ArrayList) MutablePair(org.apache.commons.lang3.tuple.MutablePair)

Example 2 with SubType

use of org.apache.pulsar.common.api.proto.CommandSubscribe.SubType in project pulsar by apache.

the class PersistentTopicTest method testChangeSubscriptionType.

@Test
public void testChangeSubscriptionType() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentSubscription sub = new PersistentSubscription(topic, "change-sub-type", cursorMock, false);
    Consumer consumer = new Consumer(sub, SubType.Exclusive, topic.getName(), 1, 0, "Cons1", true, serverCnx, "myrole-1", Collections.emptyMap(), false, InitialPosition.Latest, new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT), MessageId.latest, DEFAULT_CONSUMER_EPOCH);
    sub.addConsumer(consumer);
    consumer.close();
    SubType previousSubType = SubType.Exclusive;
    for (SubType subType : Lists.newArrayList(SubType.Shared, SubType.Failover, SubType.Key_Shared, SubType.Exclusive)) {
        Dispatcher previousDispatcher = sub.getDispatcher();
        consumer = new Consumer(sub, subType, topic.getName(), 1, 0, "Cons1", true, serverCnx, "myrole-1", Collections.emptyMap(), false, InitialPosition.Latest, new KeySharedMeta().setKeySharedMode(KeySharedMode.AUTO_SPLIT), MessageId.latest, DEFAULT_CONSUMER_EPOCH);
        sub.addConsumer(consumer);
        assertTrue(sub.getDispatcher().isConsumerConnected());
        assertFalse(sub.getDispatcher().isClosed());
        assertEquals(sub.getDispatcher().getType(), subType);
        assertFalse(previousDispatcher.isConsumerConnected());
        assertTrue(previousDispatcher.isClosed());
        assertEquals(previousDispatcher.getType(), previousSubType);
        consumer.close();
        previousSubType = subType;
    }
}
Also used : SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) PersistentDispatcherSingleActiveConsumer(org.apache.pulsar.broker.service.persistent.PersistentDispatcherSingleActiveConsumer) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription) Test(org.testng.annotations.Test)

Example 3 with SubType

use of org.apache.pulsar.common.api.proto.CommandSubscribe.SubType in project pulsar by apache.

the class PersistentTopicsBase method internalSetSubscriptionTypesEnabled.

protected CompletableFuture<Void> internalSetSubscriptionTypesEnabled(Set<SubscriptionType> subscriptionTypesEnabled, boolean isGlobal) {
    List<SubType> subTypes = Lists.newArrayList();
    subscriptionTypesEnabled.forEach(subscriptionType -> subTypes.add(SubType.valueOf(subscriptionType.name())));
    return getTopicPoliciesAsyncWithRetry(topicName, isGlobal).thenCompose(op -> {
        TopicPolicies topicPolicies = op.orElseGet(TopicPolicies::new);
        topicPolicies.setSubscriptionTypesEnabled(subTypes);
        topicPolicies.setIsGlobal(isGlobal);
        return pulsar().getTopicPoliciesService().updateTopicPoliciesAsync(topicName, topicPolicies);
    });
}
Also used : SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) InactiveTopicPolicies(org.apache.pulsar.common.policies.data.InactiveTopicPolicies) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies)

Example 4 with SubType

use of org.apache.pulsar.common.api.proto.CommandSubscribe.SubType in project pulsar by apache.

the class ServerCnx method handleSubscribe.

@Override
protected void handleSubscribe(final CommandSubscribe subscribe) {
    checkArgument(state == State.Connected);
    final long requestId = subscribe.getRequestId();
    final long consumerId = subscribe.getConsumerId();
    TopicName topicName = validateTopicName(subscribe.getTopic(), requestId, subscribe);
    if (topicName == null) {
        return;
    }
    if (log.isDebugEnabled()) {
        log.debug("[{}] Handle subscribe command: auth role = {}, original auth role = {}", remoteAddress, authRole, originalPrincipal);
    }
    if (invalidOriginalPrincipal(originalPrincipal)) {
        final String msg = "Valid Proxy Client role should be provided while subscribing ";
        log.warn("[{}] {} with role {} and proxyClientAuthRole {} on topic {}", remoteAddress, msg, authRole, originalPrincipal, topicName);
        commandSender.sendErrorResponse(requestId, ServerError.AuthorizationError, msg);
        return;
    }
    final String subscriptionName = subscribe.getSubscription();
    final SubType subType = subscribe.getSubType();
    final String consumerName = subscribe.hasConsumerName() ? subscribe.getConsumerName() : "";
    final boolean isDurable = subscribe.isDurable();
    final MessageIdImpl startMessageId = subscribe.hasStartMessageId() ? new BatchMessageIdImpl(subscribe.getStartMessageId().getLedgerId(), subscribe.getStartMessageId().getEntryId(), subscribe.getStartMessageId().getPartition(), subscribe.getStartMessageId().getBatchIndex()) : null;
    final int priorityLevel = subscribe.hasPriorityLevel() ? subscribe.getPriorityLevel() : 0;
    final boolean readCompacted = subscribe.hasReadCompacted() && subscribe.isReadCompacted();
    final Map<String, String> metadata = CommandUtils.metadataFromCommand(subscribe);
    final InitialPosition initialPosition = subscribe.getInitialPosition();
    final long startMessageRollbackDurationSec = subscribe.hasStartMessageRollbackDurationSec() ? subscribe.getStartMessageRollbackDurationSec() : -1;
    final SchemaData schema = subscribe.hasSchema() ? getSchema(subscribe.getSchema()) : null;
    final boolean isReplicated = subscribe.hasReplicateSubscriptionState() && subscribe.isReplicateSubscriptionState();
    final boolean forceTopicCreation = subscribe.isForceTopicCreation();
    final KeySharedMeta keySharedMeta = subscribe.hasKeySharedMeta() ? new KeySharedMeta().copyFrom(subscribe.getKeySharedMeta()) : emptyKeySharedMeta;
    CompletableFuture<Boolean> isAuthorizedFuture = isTopicOperationAllowed(topicName, subscriptionName, TopicOperation.CONSUME);
    // Make sure the consumer future is put into the consumers map first to avoid the same consumer
    // epoch using different consumer futures, and only remove the consumer future from the map
    // if subscribe failed .
    CompletableFuture<Consumer> consumerFuture = new CompletableFuture<>();
    CompletableFuture<Consumer> existingConsumerFuture = consumers.putIfAbsent(consumerId, consumerFuture);
    isAuthorizedFuture.thenApply(isAuthorized -> {
        if (isAuthorized) {
            if (log.isDebugEnabled()) {
                log.debug("[{}] Client is authorized to subscribe with role {}", remoteAddress, getPrincipal());
            }
            log.info("[{}] Subscribing on topic {} / {}", remoteAddress, topicName, subscriptionName);
            try {
                Metadata.validateMetadata(metadata);
            } catch (IllegalArgumentException iae) {
                final String msg = iae.getMessage();
                consumers.remove(consumerId, consumerFuture);
                commandSender.sendErrorResponse(requestId, ServerError.MetadataError, msg);
                return null;
            }
            if (existingConsumerFuture != null) {
                if (existingConsumerFuture.isDone() && !existingConsumerFuture.isCompletedExceptionally()) {
                    Consumer consumer = existingConsumerFuture.getNow(null);
                    log.info("[{}] Consumer with the same id is already created:" + " consumerId={}, consumer={}", remoteAddress, consumerId, consumer);
                    commandSender.sendSuccessResponse(requestId);
                    return null;
                } else {
                    // There was an early request to create a consumer with same consumerId. This can happen
                    // when
                    // client timeout is lower the broker timeouts. We need to wait until the previous
                    // consumer
                    // creation request either complete or fails.
                    log.warn("[{}][{}][{}] Consumer with id is already present on the connection," + " consumerId={}", remoteAddress, topicName, subscriptionName, consumerId);
                    ServerError error = null;
                    if (!existingConsumerFuture.isDone()) {
                        error = ServerError.ServiceNotReady;
                    } else {
                        error = getErrorCode(existingConsumerFuture);
                        consumers.remove(consumerId, existingConsumerFuture);
                    }
                    commandSender.sendErrorResponse(requestId, error, "Consumer is already present on the connection");
                    return null;
                }
            }
            boolean createTopicIfDoesNotExist = forceTopicCreation && service.isAllowAutoTopicCreation(topicName.toString());
            service.getTopic(topicName.toString(), createTopicIfDoesNotExist).thenCompose(optTopic -> {
                if (!optTopic.isPresent()) {
                    return FutureUtil.failedFuture(new TopicNotFoundException("Topic " + topicName + " does not exist"));
                }
                Topic topic = optTopic.get();
                boolean rejectSubscriptionIfDoesNotExist = isDurable && !service.isAllowAutoSubscriptionCreation(topicName.toString()) && !topic.getSubscriptions().containsKey(subscriptionName) && topic.isPersistent();
                if (rejectSubscriptionIfDoesNotExist) {
                    return FutureUtil.failedFuture(new SubscriptionNotFoundException("Subscription does not exist"));
                }
                long consumerEpoch = DEFAULT_CONSUMER_EPOCH;
                if (subscribe.hasConsumerEpoch()) {
                    consumerEpoch = subscribe.getConsumerEpoch();
                }
                SubscriptionOption option = SubscriptionOption.builder().cnx(ServerCnx.this).subscriptionName(subscriptionName).consumerId(consumerId).subType(subType).priorityLevel(priorityLevel).consumerName(consumerName).isDurable(isDurable).startMessageId(startMessageId).metadata(metadata).readCompacted(readCompacted).initialPosition(initialPosition).startMessageRollbackDurationSec(startMessageRollbackDurationSec).replicatedSubscriptionStateArg(isReplicated).keySharedMeta(keySharedMeta).subscriptionProperties(SubscriptionOption.getPropertiesMap(subscribe.getSubscriptionPropertiesList())).consumerEpoch(consumerEpoch).build();
                if (schema != null) {
                    return topic.addSchemaIfIdleOrCheckCompatible(schema).thenCompose(v -> topic.subscribe(option));
                } else {
                    return topic.subscribe(option);
                }
            }).thenAccept(consumer -> {
                if (consumerFuture.complete(consumer)) {
                    log.info("[{}] Created subscription on topic {} / {}", remoteAddress, topicName, subscriptionName);
                    commandSender.sendSuccessResponse(requestId);
                    if (getBrokerService().getInterceptor() != null) {
                        getBrokerService().getInterceptor().consumerCreated(this, consumer, metadata);
                    }
                } else {
                    // The consumer future was completed before by a close command
                    try {
                        consumer.close();
                        log.info("[{}] Cleared consumer created after timeout on client side {}", remoteAddress, consumer);
                    } catch (BrokerServiceException e) {
                        log.warn("[{}] Error closing consumer created" + " after timeout on client side {}: {}", remoteAddress, consumer, e.getMessage());
                    }
                    consumers.remove(consumerId, consumerFuture);
                }
            }).exceptionally(exception -> {
                if (exception.getCause() instanceof ConsumerBusyException) {
                    if (log.isDebugEnabled()) {
                        log.debug("[{}][{}][{}] Failed to create consumer because exclusive consumer" + " is already connected: {}", remoteAddress, topicName, subscriptionName, exception.getCause().getMessage());
                    }
                } else if (exception.getCause() instanceof BrokerServiceException) {
                    log.warn("[{}][{}][{}] Failed to create consumer: consumerId={}, {}", remoteAddress, topicName, subscriptionName, consumerId, exception.getCause().getMessage());
                } else {
                    log.warn("[{}][{}][{}] Failed to create consumer: consumerId={}, {}", remoteAddress, topicName, subscriptionName, consumerId, exception.getCause().getMessage(), exception);
                }
                // back to client, only if not completed already.
                if (consumerFuture.completeExceptionally(exception)) {
                    commandSender.sendErrorResponse(requestId, BrokerServiceException.getClientErrorCode(exception), exception.getCause().getMessage());
                }
                consumers.remove(consumerId, consumerFuture);
                return null;
            });
        } else {
            String msg = "Client is not authorized to subscribe";
            log.warn("[{}] {} with role {}", remoteAddress, msg, getPrincipal());
            consumers.remove(consumerId, consumerFuture);
            ctx.writeAndFlush(Commands.newError(requestId, ServerError.AuthorizationError, msg));
        }
        return null;
    }).exceptionally(ex -> {
        logAuthException(remoteAddress, "subscribe", getPrincipal(), Optional.of(topicName), ex);
        consumers.remove(consumerId, consumerFuture);
        commandSender.sendErrorResponse(requestId, ServerError.AuthorizationError, ex.getMessage());
        return null;
    });
}
Also used : CommandAuthResponse(org.apache.pulsar.common.api.proto.CommandAuthResponse) CommandUnsubscribe(org.apache.pulsar.common.api.proto.CommandUnsubscribe) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) CommandProducer(org.apache.pulsar.common.api.proto.CommandProducer) MessageIdData(org.apache.pulsar.common.api.proto.MessageIdData) ByteBufPair(org.apache.pulsar.common.protocol.ByteBufPair) StringUtils(org.apache.commons.lang3.StringUtils) ProtocolVersion(org.apache.pulsar.common.api.proto.ProtocolVersion) TxnID(org.apache.pulsar.client.api.transaction.TxnID) MLTransactionMetadataStore(org.apache.pulsar.transaction.coordinator.impl.MLTransactionMetadataStore) TopicOperation(org.apache.pulsar.common.policies.data.TopicOperation) MutableLong(org.apache.commons.lang3.mutable.MutableLong) Map(java.util.Map) BrokerInterceptor(org.apache.pulsar.broker.intercept.BrokerInterceptor) RestException(org.apache.pulsar.broker.web.RestException) NamespaceOperation(org.apache.pulsar.common.policies.data.NamespaceOperation) BaseCommand(org.apache.pulsar.common.api.proto.BaseCommand) CommandAck(org.apache.pulsar.common.api.proto.CommandAck) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) InterceptException(org.apache.pulsar.common.intercept.InterceptException) CommandFlow(org.apache.pulsar.common.api.proto.CommandFlow) CommandConsumerStats(org.apache.pulsar.common.api.proto.CommandConsumerStats) ServerError(org.apache.pulsar.common.api.proto.ServerError) Set(java.util.Set) CommandNewTxn(org.apache.pulsar.common.api.proto.CommandNewTxn) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) SafeRun(org.apache.bookkeeper.mledger.util.SafeRun) SslHandler(io.netty.handler.ssl.SslHandler) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) ExceptionUtils(org.apache.commons.lang3.exception.ExceptionUtils) ClientCnx(org.apache.pulsar.client.impl.ClientCnx) ChannelOption(io.netty.channel.ChannelOption) SchemaRegistryService(org.apache.pulsar.broker.service.schema.SchemaRegistryService) AuthenticationState(org.apache.pulsar.broker.authentication.AuthenticationState) CommandGetTopicsOfNamespace(org.apache.pulsar.common.api.proto.CommandGetTopicsOfNamespace) TopicNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.TopicNotFoundException) SchemaType(org.apache.pulsar.common.schema.SchemaType) Commands(org.apache.pulsar.common.protocol.Commands) CommandCloseProducer(org.apache.pulsar.common.api.proto.CommandCloseProducer) Strings(com.google.common.base.Strings) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) ServerMetadataException(org.apache.pulsar.broker.service.BrokerServiceException.ServerMetadataException) SSLSession(javax.net.ssl.SSLSession) CommandGetOrCreateSchema(org.apache.pulsar.common.api.proto.CommandGetOrCreateSchema) ProtocolVersion.v5(org.apache.pulsar.common.api.proto.ProtocolVersion.v5) CommandGetSchema(org.apache.pulsar.common.api.proto.CommandGetSchema) SchemaInfoUtil(org.apache.pulsar.client.impl.schema.SchemaInfoUtil) CommandRedeliverUnacknowledgedMessages(org.apache.pulsar.common.api.proto.CommandRedeliverUnacknowledgedMessages) Metadata(org.apache.pulsar.common.naming.Metadata) Promise(io.netty.util.concurrent.Promise) AuthenticationProvider(org.apache.pulsar.broker.authentication.AuthenticationProvider) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) lombok.val(lombok.val) PulsarService(org.apache.pulsar.broker.PulsarService) KeySharedMode(org.apache.pulsar.common.api.proto.KeySharedMode) CommandEndTxnOnSubscription(org.apache.pulsar.common.api.proto.CommandEndTxnOnSubscription) CommandConnect(org.apache.pulsar.common.api.proto.CommandConnect) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) ProducerAccessMode(org.apache.pulsar.common.api.proto.ProducerAccessMode) CommandLookupTopic(org.apache.pulsar.common.api.proto.CommandLookupTopic) MutableInt(org.apache.commons.lang3.mutable.MutableInt) SocketAddress(java.net.SocketAddress) CommandAddPartitionToTxn(org.apache.pulsar.common.api.proto.CommandAddPartitionToTxn) LoggerFactory(org.slf4j.LoggerFactory) AuthData(org.apache.pulsar.common.api.AuthData) Exceptions(org.apache.pulsar.functions.utils.Exceptions) AuthenticationException(javax.naming.AuthenticationException) KeyValue(org.apache.pulsar.common.api.proto.KeyValue) PersistentTopicsBase.unsafeGetPartitionedTopicMetadataAsync(org.apache.pulsar.broker.admin.impl.PersistentTopicsBase.unsafeGetPartitionedTopicMetadataAsync) CommandEndTxnOnPartition(org.apache.pulsar.common.api.proto.CommandEndTxnOnPartition) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) BacklogQuotaType(org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType) Gauge(io.prometheus.client.Gauge) TxnAction(org.apache.pulsar.common.api.proto.TxnAction) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) Schema(org.apache.pulsar.common.api.proto.Schema) TransactionMetadataStoreService(org.apache.pulsar.broker.TransactionMetadataStoreService) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) IdentityHashMap(java.util.IdentityHashMap) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException) Position(org.apache.bookkeeper.mledger.Position) InetSocketAddress(java.net.InetSocketAddress) Collectors(java.util.stream.Collectors) CommandEndTxn(org.apache.pulsar.common.api.proto.CommandEndTxn) AuthenticationDataSource(org.apache.pulsar.broker.authentication.AuthenticationDataSource) Objects(java.util.Objects) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) FutureUtil(org.apache.pulsar.common.util.FutureUtil) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription) Optional(java.util.Optional) IncompatibleSchemaException(org.apache.pulsar.broker.service.schema.exceptions.IncompatibleSchemaException) PulsarHandler(org.apache.pulsar.common.protocol.PulsarHandler) CommandTcClientConnectRequest(org.apache.pulsar.common.api.proto.CommandTcClientConnectRequest) TopicName(org.apache.pulsar.common.naming.TopicName) CommandAddSubscriptionToTxn(org.apache.pulsar.common.api.proto.CommandAddSubscriptionToTxn) Entry(org.apache.bookkeeper.mledger.Entry) CommandSend(org.apache.pulsar.common.api.proto.CommandSend) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) TopicLookupBase.lookupTopicAsync(org.apache.pulsar.broker.lookup.TopicLookupBase.lookupTopicAsync) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) CommandSubscribe(org.apache.pulsar.common.api.proto.CommandSubscribe) ChannelHandlerContext(io.netty.channel.ChannelHandlerContext) ByteBuf(io.netty.buffer.ByteBuf) CoordinatorException(org.apache.pulsar.transaction.coordinator.exceptions.CoordinatorException) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) CommandPartitionedTopicMetadata(org.apache.pulsar.common.api.proto.CommandPartitionedTopicMetadata) CommandSeek(org.apache.pulsar.common.api.proto.CommandSeek) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) NoSuchElementException(java.util.NoSuchElementException) CommandGetLastMessageId(org.apache.pulsar.common.api.proto.CommandGetLastMessageId) CommandUtils(org.apache.pulsar.common.protocol.CommandUtils) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) AuthenticationDataCommand(org.apache.pulsar.broker.authentication.AuthenticationDataCommand) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) Logger(org.slf4j.Logger) Semaphore(java.util.concurrent.Semaphore) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) FeatureFlags(org.apache.pulsar.common.api.proto.FeatureFlags) CommandCloseConsumer(org.apache.pulsar.common.api.proto.CommandCloseConsumer) SchemaVersion(org.apache.pulsar.common.protocol.schema.SchemaVersion) Commands.newLookupErrorResponse(org.apache.pulsar.common.protocol.Commands.newLookupErrorResponse) TimeUnit(java.util.concurrent.TimeUnit) HAProxyMessage(io.netty.handler.codec.haproxy.HAProxyMessage) ConcurrentLongHashMap(org.apache.pulsar.common.util.collections.ConcurrentLongHashMap) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) ChannelHandler(io.netty.channel.ChannelHandler) VisibleForTesting(com.google.common.annotations.VisibleForTesting) TransactionCoordinatorID(org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID) Collections(java.util.Collections) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) TopicNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.TopicNotFoundException) CompletableFuture(java.util.concurrent.CompletableFuture) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) CommandCloseConsumer(org.apache.pulsar.common.api.proto.CommandCloseConsumer) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) CommandLookupTopic(org.apache.pulsar.common.api.proto.CommandLookupTopic) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) ServerError(org.apache.pulsar.common.api.proto.ServerError) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) TopicName(org.apache.pulsar.common.naming.TopicName) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException)

Example 5 with SubType

use of org.apache.pulsar.common.api.proto.CommandSubscribe.SubType in project pulsar by apache.

the class Consumer method individualAckNormal.

// this method is for individual ack not carry the transaction
private CompletableFuture<Void> individualAckNormal(CommandAck ack, Map<String, Long> properties) {
    List<Position> positionsAcked = new ArrayList<>();
    for (int i = 0; i < ack.getMessageIdsCount(); i++) {
        MessageIdData msgId = ack.getMessageIdAt(i);
        PositionImpl position;
        long ackedCount = 0;
        long batchSize = getBatchSize(msgId);
        Consumer ackOwnerConsumer = getAckOwnerConsumer(msgId.getLedgerId(), msgId.getEntryId());
        if (msgId.getAckSetsCount() > 0) {
            long[] ackSets = new long[msgId.getAckSetsCount()];
            for (int j = 0; j < msgId.getAckSetsCount(); j++) {
                ackSets[j] = msgId.getAckSetAt(j);
            }
            position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId(), ackSets);
            ackedCount = getAckedCountForBatchIndexLevelEnabled(position, batchSize, ackSets);
            if (isTransactionEnabled()) {
                // sync the batch position bit set point, in order to delete the position in pending acks
                if (Subscription.isIndividualAckMode(subType)) {
                    ((PersistentSubscription) subscription).syncBatchPositionBitSetForPendingAck(position);
                }
            }
        } else {
            position = PositionImpl.get(msgId.getLedgerId(), msgId.getEntryId());
            ackedCount = getAckedCountForMsgIdNoAckSets(batchSize, position);
        }
        addAndGetUnAckedMsgs(ackOwnerConsumer, -(int) ackedCount);
        positionsAcked.add(position);
        checkCanRemovePendingAcksAndHandle(position, msgId);
        checkAckValidationError(ack, position);
    }
    subscription.acknowledgeMessage(positionsAcked, AckType.Individual, properties);
    CompletableFuture<Void> completableFuture = new CompletableFuture<>();
    completableFuture.complete(null);
    if (isTransactionEnabled() && Subscription.isIndividualAckMode(subType)) {
        completableFuture.whenComplete((v, e) -> positionsAcked.forEach(position -> {
            // the bit set is empty in pending ack handle.
            if (((PositionImpl) position).getAckSet() != null) {
                if (((PersistentSubscription) subscription).checkIsCanDeleteConsumerPendingAck((PositionImpl) position)) {
                    removePendingAcks((PositionImpl) position);
                }
            }
        }));
    }
    return completableFuture;
}
Also used : LongAdder(java.util.concurrent.atomic.LongAdder) AtomicIntegerFieldUpdater(java.util.concurrent.atomic.AtomicIntegerFieldUpdater) Setter(lombok.Setter) MutableInt(org.apache.commons.lang3.mutable.MutableInt) TopicName(org.apache.pulsar.common.naming.TopicName) LongPair(org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap.LongPair) Getter(lombok.Getter) AckType(org.apache.pulsar.common.api.proto.CommandAck.AckType) MessageIdData(org.apache.pulsar.common.api.proto.MessageIdData) Entry(org.apache.bookkeeper.mledger.Entry) LoggerFactory(org.slf4j.LoggerFactory) CompletableFuture(java.util.concurrent.CompletableFuture) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) ArrayList(java.util.ArrayList) Commands(org.apache.pulsar.common.protocol.Commands) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) TxnID(org.apache.pulsar.client.api.transaction.TxnID) Lists(com.google.common.collect.Lists) KeyLongValue(org.apache.pulsar.common.api.proto.KeyLongValue) DateFormatter(org.apache.pulsar.common.util.DateFormatter) MutablePair(org.apache.commons.lang3.tuple.MutablePair) Map(java.util.Map) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) CommandAck(org.apache.pulsar.common.api.proto.CommandAck) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) Logger(org.slf4j.Logger) Promise(io.netty.util.concurrent.Promise) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) TransactionConflictException(org.apache.pulsar.transaction.common.exception.TransactionConflictException) BitSetRecyclable(org.apache.pulsar.common.util.collections.BitSetRecyclable) MoreObjects(com.google.common.base.MoreObjects) Position(org.apache.bookkeeper.mledger.Position) Collectors(java.util.stream.Collectors) Objects(java.util.Objects) List(java.util.List) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) FutureUtil(org.apache.pulsar.common.util.FutureUtil) MessageId(org.apache.pulsar.client.api.MessageId) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription) ConcurrentLongLongPairHashMap(org.apache.pulsar.common.util.collections.ConcurrentLongLongPairHashMap) BitSet(java.util.BitSet) Future(io.netty.util.concurrent.Future) Collections(java.util.Collections) Rate(org.apache.pulsar.common.stats.Rate) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) Position(org.apache.bookkeeper.mledger.Position) MessageIdData(org.apache.pulsar.common.api.proto.MessageIdData) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) ArrayList(java.util.ArrayList) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription) CompletableFuture(java.util.concurrent.CompletableFuture)

Aggregations

SubType (org.apache.pulsar.common.api.proto.CommandSubscribe.SubType)7 PositionImpl (org.apache.bookkeeper.mledger.impl.PositionImpl)4 PersistentSubscription (org.apache.pulsar.broker.service.persistent.PersistentSubscription)4 PersistentTopic (org.apache.pulsar.broker.service.persistent.PersistentTopic)4 KeySharedMeta (org.apache.pulsar.common.api.proto.KeySharedMeta)4 ConsumerStatsImpl (org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl)4 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)3 Promise (io.netty.util.concurrent.Promise)3 Collections (java.util.Collections)3 List (java.util.List)3 Map (java.util.Map)3 Objects (java.util.Objects)3 CompletableFuture (java.util.concurrent.CompletableFuture)3 Collectors (java.util.stream.Collectors)3 Entry (org.apache.bookkeeper.mledger.Entry)3 Position (org.apache.bookkeeper.mledger.Position)3 MutableInt (org.apache.commons.lang3.mutable.MutableInt)3 TxnID (org.apache.pulsar.client.api.transaction.TxnID)3 CommandAck (org.apache.pulsar.common.api.proto.CommandAck)3 InitialPosition (org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition)3