Search in sources :

Example 1 with PartitionedTopicResources

use of org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources in project pulsar by apache.

the class BrokerService method createDefaultPartitionedTopicAsync.

@SuppressWarnings("deprecation")
private CompletableFuture<PartitionedTopicMetadata> createDefaultPartitionedTopicAsync(TopicName topicName) {
    final int defaultNumPartitions = pulsar.getBrokerService().getDefaultNumPartitions(topicName);
    final int maxPartitions = pulsar().getConfig().getMaxNumPartitionsPerPartitionedTopic();
    checkArgument(defaultNumPartitions > 0, "Default number of partitions should be more than 0");
    checkArgument(maxPartitions <= 0 || defaultNumPartitions <= maxPartitions, "Number of partitions should be less than or equal to " + maxPartitions);
    PartitionedTopicMetadata configMetadata = new PartitionedTopicMetadata(defaultNumPartitions);
    return checkMaxTopicsPerNamespace(topicName, defaultNumPartitions).thenCompose(__ -> {
        PartitionedTopicResources partitionResources = pulsar.getPulsarResources().getNamespaceResources().getPartitionedTopicResources();
        return partitionResources.createPartitionedTopicAsync(topicName, configMetadata).thenApply(v -> {
            log.info("partitioned metadata successfully created for {}", topicName);
            return configMetadata;
        });
    });
}
Also used : PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata)

Example 2 with PartitionedTopicResources

use of org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources in project pulsar by apache.

the class PersistentTopic method tryToDeletePartitionedMetadata.

private CompletableFuture<Void> tryToDeletePartitionedMetadata() {
    if (TopicName.get(topic).isPartitioned() && !deletePartitionedTopicMetadataWhileInactive()) {
        return CompletableFuture.completedFuture(null);
    }
    TopicName topicName = TopicName.get(TopicName.get(topic).getPartitionedTopicName());
    PartitionedTopicResources partitionedTopicResources = getBrokerService().pulsar().getPulsarResources().getNamespaceResources().getPartitionedTopicResources();
    return partitionedTopicResources.partitionedTopicExistsAsync(topicName).thenCompose(partitionedTopicExist -> {
        if (!partitionedTopicExist) {
            return CompletableFuture.completedFuture(null);
        } else {
            return getBrokerService().fetchPartitionedTopicMetadataAsync(topicName).thenCompose((metadata -> {
                List<CompletableFuture<Boolean>> persistentTopicExists = new ArrayList<>(metadata.partitions);
                for (int i = 0; i < metadata.partitions; i++) {
                    persistentTopicExists.add(brokerService.getPulsar().getPulsarResources().getTopicResources().persistentTopicExists(topicName.getPartition(i)));
                }
                List<CompletableFuture<Boolean>> unmodifiablePersistentTopicExists = Collections.unmodifiableList(persistentTopicExists);
                return FutureUtil.waitForAll(unmodifiablePersistentTopicExists).thenCompose(unused -> {
                    // make sure all sub partitions were deleted after all future complete
                    Optional<Boolean> anyExistPartition = unmodifiablePersistentTopicExists.stream().map(CompletableFuture::join).filter(topicExist -> topicExist).findAny();
                    if (anyExistPartition.isPresent()) {
                        log.error("[{}] Delete topic metadata failed because" + " another partition exist.", topicName);
                        throw new UnsupportedOperationException(String.format("Another partition exists for [%s].", topicName));
                    } else {
                        return partitionedTopicResources.deletePartitionedTopicAsync(topicName);
                    }
                });
            }));
        }
    });
}
Also used : EventsTopicNames(org.apache.pulsar.common.events.EventsTopicNames) SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) Topic(org.apache.pulsar.broker.service.Topic) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) StringUtils(org.apache.commons.lang3.StringUtils) MetadataNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) TxnID(org.apache.pulsar.client.api.transaction.TxnID) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) AlreadyRunningException(org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException) Map(java.util.Map) StatsOutputStream(org.apache.pulsar.utils.StatsOutputStream) EnumSet(java.util.EnumSet) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) UpdatePropertiesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.UpdatePropertiesCallback) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) ManagedLedgerAlreadyClosedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException) CancellationException(java.util.concurrent.CancellationException) ManagedCursorContainer(org.apache.bookkeeper.mledger.impl.ManagedCursorContainer) Set(java.util.Set) BrokerService(org.apache.pulsar.broker.service.BrokerService) NamespaceStats(org.apache.pulsar.broker.stats.NamespaceStats) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) OffloadProcessStatus(org.apache.pulsar.client.admin.OffloadProcessStatus) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) TransactionPendingAckStats(org.apache.pulsar.common.policies.data.TransactionPendingAckStats) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) TopicTerminatedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) ManagedLedgerTerminatedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) BookieId(org.apache.bookkeeper.net.BookieId) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) TransactionInPendingAckStats(org.apache.pulsar.common.policies.data.TransactionInPendingAckStats) Subscription(org.apache.pulsar.broker.service.Subscription) LedgerInfo(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo) Consumer(org.apache.pulsar.broker.service.Consumer) TerminateCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback) ArrayList(java.util.ArrayList) TopicClosedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException) Commands(org.apache.pulsar.common.protocol.Commands) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Lists(com.google.common.collect.Lists) CursorStats(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats) DateFormatter(org.apache.pulsar.common.util.DateFormatter) TopicBacklogQuotaExceededException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBacklogQuotaExceededException) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) CompactorMXBean(org.apache.pulsar.compaction.CompactorMXBean) Dispatcher(org.apache.pulsar.broker.service.Dispatcher) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) BrokerServiceException(org.apache.pulsar.broker.service.BrokerServiceException) StreamingStats(org.apache.pulsar.broker.service.StreamingStats) AbstractReplicator(org.apache.pulsar.broker.service.AbstractReplicator) TopicFencedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException) EventsTopicNames.checkTopicIsEventsNames(org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Compactor(org.apache.pulsar.compaction.Compactor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) Codec(org.apache.pulsar.common.util.Codec) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) ScheduledFuture(java.util.concurrent.ScheduledFuture) COMPACTION_SUBSCRIPTION(org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION) BookkeeperSchemaStorage(org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage) BiFunction(java.util.function.BiFunction) UnsupportedVersionException(org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) LoggerFactory(org.slf4j.LoggerFactory) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) Type(org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type) BacklogQuotaType(org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) TxnAction(org.apache.pulsar.common.api.proto.TxnAction) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) TransactionInBufferStats(org.apache.pulsar.common.policies.data.TransactionInBufferStats) ReplicationMetrics(org.apache.pulsar.broker.stats.ReplicationMetrics) CompletionException(java.util.concurrent.CompletionException) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) CompactedTopicContext(org.apache.pulsar.compaction.CompactedTopicContext) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) InactiveTopicDeleteMode(org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) MLPendingAckStore(org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore) Optional(java.util.Optional) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) LedgerMetadata(org.apache.bookkeeper.client.api.LedgerMetadata) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) LongAdder(java.util.concurrent.atomic.LongAdder) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) CommandSubscribe(org.apache.pulsar.common.api.proto.CommandSubscribe) SubscriptionOption(org.apache.pulsar.broker.service.SubscriptionOption) ByteBuf(io.netty.buffer.ByteBuf) CompactedTopicImpl(org.apache.pulsar.compaction.CompactedTopicImpl) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) TransactionBufferDisable(org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable) LongRunningProcessStatus(org.apache.pulsar.client.admin.LongRunningProcessStatus) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) AbstractTopic(org.apache.pulsar.broker.service.AbstractTopic) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) MapUtils(org.apache.commons.collections4.MapUtils) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) TransportCnx(org.apache.pulsar.broker.service.TransportCnx) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) Logger(org.slf4j.Logger) CompactedTopic(org.apache.pulsar.compaction.CompactedTopic) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) OffloadCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OffloadCallback) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) Producer(org.apache.pulsar.broker.service.Producer) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) Maps(com.google.common.collect.Maps) TransactionBufferStats(org.apache.pulsar.common.policies.data.TransactionBufferStats) TransactionBuffer(org.apache.pulsar.broker.transaction.buffer.TransactionBuffer) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) MessageId(org.apache.pulsar.client.api.MessageId) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) Clock(java.time.Clock) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) Replicator(org.apache.pulsar.broker.service.Replicator) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) Optional(java.util.Optional) ArrayList(java.util.ArrayList) List(java.util.List) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TopicName(org.apache.pulsar.common.naming.TopicName)

Example 3 with PartitionedTopicResources

use of org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources in project pulsar by yahoo.

the class PersistentTopic method tryToDeletePartitionedMetadata.

private CompletableFuture<Void> tryToDeletePartitionedMetadata() {
    if (TopicName.get(topic).isPartitioned() && !deletePartitionedTopicMetadataWhileInactive()) {
        return CompletableFuture.completedFuture(null);
    }
    TopicName topicName = TopicName.get(TopicName.get(topic).getPartitionedTopicName());
    PartitionedTopicResources partitionedTopicResources = getBrokerService().pulsar().getPulsarResources().getNamespaceResources().getPartitionedTopicResources();
    return partitionedTopicResources.partitionedTopicExistsAsync(topicName).thenCompose(partitionedTopicExist -> {
        if (!partitionedTopicExist) {
            return CompletableFuture.completedFuture(null);
        } else {
            return getBrokerService().fetchPartitionedTopicMetadataAsync(topicName).thenCompose((metadata -> {
                List<CompletableFuture<Boolean>> persistentTopicExists = new ArrayList<>(metadata.partitions);
                for (int i = 0; i < metadata.partitions; i++) {
                    persistentTopicExists.add(brokerService.getPulsar().getPulsarResources().getTopicResources().persistentTopicExists(topicName.getPartition(i)));
                }
                List<CompletableFuture<Boolean>> unmodifiablePersistentTopicExists = Collections.unmodifiableList(persistentTopicExists);
                return FutureUtil.waitForAll(unmodifiablePersistentTopicExists).thenCompose(unused -> {
                    // make sure all sub partitions were deleted after all future complete
                    Optional<Boolean> anyExistPartition = unmodifiablePersistentTopicExists.stream().map(CompletableFuture::join).filter(topicExist -> topicExist).findAny();
                    if (anyExistPartition.isPresent()) {
                        log.error("[{}] Delete topic metadata failed because" + " another partition exist.", topicName);
                        throw new UnsupportedOperationException(String.format("Another partition exists for [%s].", topicName));
                    } else {
                        return partitionedTopicResources.deletePartitionedTopicAsync(topicName);
                    }
                });
            }));
        }
    });
}
Also used : SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) Topic(org.apache.pulsar.broker.service.Topic) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) StringUtils(org.apache.commons.lang3.StringUtils) SubscribeRate(org.apache.pulsar.common.policies.data.SubscribeRate) MetadataNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) TxnID(org.apache.pulsar.client.api.transaction.TxnID) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) AlreadyRunningException(org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException) Map(java.util.Map) StatsOutputStream(org.apache.pulsar.utils.StatsOutputStream) EnumSet(java.util.EnumSet) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) UpdatePropertiesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.UpdatePropertiesCallback) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) ManagedLedgerAlreadyClosedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException) CancellationException(java.util.concurrent.CancellationException) ManagedCursorContainer(org.apache.bookkeeper.mledger.impl.ManagedCursorContainer) Set(java.util.Set) BrokerService(org.apache.pulsar.broker.service.BrokerService) NamespaceStats(org.apache.pulsar.broker.stats.NamespaceStats) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) OffloadProcessStatus(org.apache.pulsar.client.admin.OffloadProcessStatus) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) TransactionPendingAckStats(org.apache.pulsar.common.policies.data.TransactionPendingAckStats) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) TopicTerminatedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) ManagedLedgerTerminatedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) BookieId(org.apache.bookkeeper.net.BookieId) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) TransactionInPendingAckStats(org.apache.pulsar.common.policies.data.TransactionInPendingAckStats) Subscription(org.apache.pulsar.broker.service.Subscription) LedgerInfo(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo) NamespaceService(org.apache.pulsar.broker.namespace.NamespaceService) Consumer(org.apache.pulsar.broker.service.Consumer) TerminateCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback) ArrayList(java.util.ArrayList) TopicClosedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException) Commands(org.apache.pulsar.common.protocol.Commands) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Lists(com.google.common.collect.Lists) CursorStats(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats) DateFormatter(org.apache.pulsar.common.util.DateFormatter) TopicBacklogQuotaExceededException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBacklogQuotaExceededException) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) CompactorMXBean(org.apache.pulsar.compaction.CompactorMXBean) Dispatcher(org.apache.pulsar.broker.service.Dispatcher) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) BrokerServiceException(org.apache.pulsar.broker.service.BrokerServiceException) StreamingStats(org.apache.pulsar.broker.service.StreamingStats) AbstractReplicator(org.apache.pulsar.broker.service.AbstractReplicator) TopicFencedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Compactor(org.apache.pulsar.compaction.Compactor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) Codec(org.apache.pulsar.common.util.Codec) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) SystemTopicNames.isEventSystemTopic(org.apache.pulsar.common.naming.SystemTopicNames.isEventSystemTopic) ScheduledFuture(java.util.concurrent.ScheduledFuture) COMPACTION_SUBSCRIPTION(org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION) BookkeeperSchemaStorage(org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage) BiFunction(java.util.function.BiFunction) UnsupportedVersionException(org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) LoggerFactory(org.slf4j.LoggerFactory) SystemTopicNames(org.apache.pulsar.common.naming.SystemTopicNames) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) Type(org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type) BacklogQuotaType(org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) TxnAction(org.apache.pulsar.common.api.proto.TxnAction) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) TransactionInBufferStats(org.apache.pulsar.common.policies.data.TransactionInBufferStats) ReplicationMetrics(org.apache.pulsar.broker.stats.ReplicationMetrics) SubscribeRateLimiter.isSubscribeRateEnabled(org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter.isSubscribeRateEnabled) CompletionException(java.util.concurrent.CompletionException) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) CompactedTopicContext(org.apache.pulsar.compaction.CompactedTopicContext) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) InactiveTopicDeleteMode(org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) MLPendingAckStore(org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore) Optional(java.util.Optional) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) LedgerMetadata(org.apache.bookkeeper.client.api.LedgerMetadata) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) LongAdder(java.util.concurrent.atomic.LongAdder) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) CommandSubscribe(org.apache.pulsar.common.api.proto.CommandSubscribe) SubscriptionOption(org.apache.pulsar.broker.service.SubscriptionOption) ByteBuf(io.netty.buffer.ByteBuf) CompactedTopicImpl(org.apache.pulsar.compaction.CompactedTopicImpl) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) TransactionBufferDisable(org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable) LongRunningProcessStatus(org.apache.pulsar.client.admin.LongRunningProcessStatus) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) AbstractTopic(org.apache.pulsar.broker.service.AbstractTopic) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) MapUtils(org.apache.commons.collections4.MapUtils) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) TransportCnx(org.apache.pulsar.broker.service.TransportCnx) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) Logger(org.slf4j.Logger) CompactedTopic(org.apache.pulsar.compaction.CompactedTopic) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) OffloadCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OffloadCallback) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) Producer(org.apache.pulsar.broker.service.Producer) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) Maps(com.google.common.collect.Maps) TransactionBufferStats(org.apache.pulsar.common.policies.data.TransactionBufferStats) TransactionBuffer(org.apache.pulsar.broker.transaction.buffer.TransactionBuffer) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) MessageId(org.apache.pulsar.client.api.MessageId) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) Clock(java.time.Clock) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) Replicator(org.apache.pulsar.broker.service.Replicator) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) Optional(java.util.Optional) ArrayList(java.util.ArrayList) List(java.util.List) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TopicName(org.apache.pulsar.common.naming.TopicName)

Example 4 with PartitionedTopicResources

use of org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources in project incubator-pulsar by apache.

the class PersistentTopic method tryToDeletePartitionedMetadata.

private CompletableFuture<Void> tryToDeletePartitionedMetadata() {
    if (TopicName.get(topic).isPartitioned() && !deletePartitionedTopicMetadataWhileInactive()) {
        return CompletableFuture.completedFuture(null);
    }
    TopicName topicName = TopicName.get(TopicName.get(topic).getPartitionedTopicName());
    PartitionedTopicResources partitionedTopicResources = getBrokerService().pulsar().getPulsarResources().getNamespaceResources().getPartitionedTopicResources();
    return partitionedTopicResources.partitionedTopicExistsAsync(topicName).thenCompose(partitionedTopicExist -> {
        if (!partitionedTopicExist) {
            return CompletableFuture.completedFuture(null);
        } else {
            return getBrokerService().fetchPartitionedTopicMetadataAsync(topicName).thenCompose((metadata -> {
                List<CompletableFuture<Boolean>> persistentTopicExists = new ArrayList<>(metadata.partitions);
                for (int i = 0; i < metadata.partitions; i++) {
                    persistentTopicExists.add(brokerService.getPulsar().getPulsarResources().getTopicResources().persistentTopicExists(topicName.getPartition(i)));
                }
                List<CompletableFuture<Boolean>> unmodifiablePersistentTopicExists = Collections.unmodifiableList(persistentTopicExists);
                return FutureUtil.waitForAll(unmodifiablePersistentTopicExists).thenCompose(unused -> {
                    // make sure all sub partitions were deleted after all future complete
                    Optional<Boolean> anyExistPartition = unmodifiablePersistentTopicExists.stream().map(CompletableFuture::join).filter(topicExist -> topicExist).findAny();
                    if (anyExistPartition.isPresent()) {
                        log.error("[{}] Delete topic metadata failed because" + " another partition exist.", topicName);
                        throw new UnsupportedOperationException(String.format("Another partition exists for [%s].", topicName));
                    } else {
                        return partitionedTopicResources.deletePartitionedTopicAsync(topicName);
                    }
                });
            }));
        }
    });
}
Also used : SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) Topic(org.apache.pulsar.broker.service.Topic) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) StringUtils(org.apache.commons.lang3.StringUtils) SubscribeRate(org.apache.pulsar.common.policies.data.SubscribeRate) MetadataNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) TxnID(org.apache.pulsar.client.api.transaction.TxnID) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) AlreadyRunningException(org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException) Map(java.util.Map) StatsOutputStream(org.apache.pulsar.utils.StatsOutputStream) EnumSet(java.util.EnumSet) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) UpdatePropertiesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.UpdatePropertiesCallback) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) ManagedLedgerAlreadyClosedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException) CancellationException(java.util.concurrent.CancellationException) ManagedCursorContainer(org.apache.bookkeeper.mledger.impl.ManagedCursorContainer) Set(java.util.Set) BrokerService(org.apache.pulsar.broker.service.BrokerService) NamespaceStats(org.apache.pulsar.broker.stats.NamespaceStats) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) OffloadProcessStatus(org.apache.pulsar.client.admin.OffloadProcessStatus) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) TransactionPendingAckStats(org.apache.pulsar.common.policies.data.TransactionPendingAckStats) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) TopicTerminatedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) ManagedLedgerTerminatedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) BookieId(org.apache.bookkeeper.net.BookieId) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) TransactionInPendingAckStats(org.apache.pulsar.common.policies.data.TransactionInPendingAckStats) Subscription(org.apache.pulsar.broker.service.Subscription) LedgerInfo(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo) NamespaceService(org.apache.pulsar.broker.namespace.NamespaceService) Consumer(org.apache.pulsar.broker.service.Consumer) TerminateCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback) ArrayList(java.util.ArrayList) TopicClosedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException) Commands(org.apache.pulsar.common.protocol.Commands) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Lists(com.google.common.collect.Lists) CursorStats(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats) DateFormatter(org.apache.pulsar.common.util.DateFormatter) TopicBacklogQuotaExceededException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBacklogQuotaExceededException) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) CompactorMXBean(org.apache.pulsar.compaction.CompactorMXBean) Dispatcher(org.apache.pulsar.broker.service.Dispatcher) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) BrokerServiceException(org.apache.pulsar.broker.service.BrokerServiceException) StreamingStats(org.apache.pulsar.broker.service.StreamingStats) AbstractReplicator(org.apache.pulsar.broker.service.AbstractReplicator) TopicFencedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Compactor(org.apache.pulsar.compaction.Compactor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) Codec(org.apache.pulsar.common.util.Codec) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) SystemTopicNames.isEventSystemTopic(org.apache.pulsar.common.naming.SystemTopicNames.isEventSystemTopic) ScheduledFuture(java.util.concurrent.ScheduledFuture) COMPACTION_SUBSCRIPTION(org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION) BookkeeperSchemaStorage(org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage) BiFunction(java.util.function.BiFunction) UnsupportedVersionException(org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) LoggerFactory(org.slf4j.LoggerFactory) SystemTopicNames(org.apache.pulsar.common.naming.SystemTopicNames) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) Type(org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type) BacklogQuotaType(org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) TxnAction(org.apache.pulsar.common.api.proto.TxnAction) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) TransactionInBufferStats(org.apache.pulsar.common.policies.data.TransactionInBufferStats) ReplicationMetrics(org.apache.pulsar.broker.stats.ReplicationMetrics) SubscribeRateLimiter.isSubscribeRateEnabled(org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter.isSubscribeRateEnabled) CompletionException(java.util.concurrent.CompletionException) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) CompactedTopicContext(org.apache.pulsar.compaction.CompactedTopicContext) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) InactiveTopicDeleteMode(org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) MLPendingAckStore(org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore) Optional(java.util.Optional) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) LedgerMetadata(org.apache.bookkeeper.client.api.LedgerMetadata) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) LongAdder(java.util.concurrent.atomic.LongAdder) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) CommandSubscribe(org.apache.pulsar.common.api.proto.CommandSubscribe) SubscriptionOption(org.apache.pulsar.broker.service.SubscriptionOption) ByteBuf(io.netty.buffer.ByteBuf) CompactedTopicImpl(org.apache.pulsar.compaction.CompactedTopicImpl) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) TransactionBufferDisable(org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable) LongRunningProcessStatus(org.apache.pulsar.client.admin.LongRunningProcessStatus) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) AbstractTopic(org.apache.pulsar.broker.service.AbstractTopic) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) MapUtils(org.apache.commons.collections4.MapUtils) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) TransportCnx(org.apache.pulsar.broker.service.TransportCnx) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) Logger(org.slf4j.Logger) CompactedTopic(org.apache.pulsar.compaction.CompactedTopic) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) OffloadCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OffloadCallback) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) Producer(org.apache.pulsar.broker.service.Producer) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) Maps(com.google.common.collect.Maps) TransactionBufferStats(org.apache.pulsar.common.policies.data.TransactionBufferStats) TransactionBuffer(org.apache.pulsar.broker.transaction.buffer.TransactionBuffer) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) MessageId(org.apache.pulsar.client.api.MessageId) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) Clock(java.time.Clock) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) Replicator(org.apache.pulsar.broker.service.Replicator) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) Optional(java.util.Optional) ArrayList(java.util.ArrayList) List(java.util.List) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) TopicName(org.apache.pulsar.common.naming.TopicName)

Example 5 with PartitionedTopicResources

use of org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources in project incubator-pulsar by apache.

the class BrokerService method createDefaultPartitionedTopicAsync.

@SuppressWarnings("deprecation")
private CompletableFuture<PartitionedTopicMetadata> createDefaultPartitionedTopicAsync(TopicName topicName) {
    if (topicName.getLocalName().contains(TopicName.PARTITIONED_TOPIC_SUFFIX)) {
        return FutureUtil.failedFuture(new PulsarServerException.InvalidTopicNameException(String.format("Invalid topic name: %s , should not contain -partition-", topicName)));
    }
    final int defaultNumPartitions = pulsar.getBrokerService().getDefaultNumPartitions(topicName);
    final int maxPartitions = pulsar().getConfig().getMaxNumPartitionsPerPartitionedTopic();
    checkArgument(defaultNumPartitions > 0, "Default number of partitions should be more than 0");
    checkArgument(maxPartitions <= 0 || defaultNumPartitions <= maxPartitions, "Number of partitions should be less than or equal to " + maxPartitions);
    PartitionedTopicMetadata configMetadata = new PartitionedTopicMetadata(defaultNumPartitions);
    return checkMaxTopicsPerNamespace(topicName, defaultNumPartitions).thenCompose(__ -> {
        PartitionedTopicResources partitionResources = pulsar.getPulsarResources().getNamespaceResources().getPartitionedTopicResources();
        return partitionResources.createPartitionedTopicAsync(topicName, configMetadata).thenApply(v -> {
            log.info("partitioned metadata successfully created for {}", topicName);
            return configMetadata;
        });
    });
}
Also used : PulsarServerException(org.apache.pulsar.broker.PulsarServerException) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata)

Aggregations

PartitionedTopicResources (org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources)6 ObjectObjectHashMap (com.carrotsearch.hppc.ObjectObjectHashMap)3 VisibleForTesting (com.google.common.annotations.VisibleForTesting)3 Preconditions.checkNotNull (com.google.common.base.Preconditions.checkNotNull)3 Lists (com.google.common.collect.Lists)3 Maps (com.google.common.collect.Maps)3 Sets (com.google.common.collect.Sets)3 ByteBuf (io.netty.buffer.ByteBuf)3 FastThreadLocal (io.netty.util.concurrent.FastThreadLocal)3 Clock (java.time.Clock)3 ArrayList (java.util.ArrayList)3 Collections (java.util.Collections)3 EnumSet (java.util.EnumSet)3 List (java.util.List)3 Map (java.util.Map)3 Optional (java.util.Optional)3 Set (java.util.Set)3 CancellationException (java.util.concurrent.CancellationException)3 CompletableFuture (java.util.concurrent.CompletableFuture)3 CompletionException (java.util.concurrent.CompletionException)3