Search in sources :

Example 1 with SystemTopic

use of org.apache.pulsar.broker.service.persistent.SystemTopic in project pulsar by apache.

the class BrokerService method createPersistentTopic.

private void createPersistentTopic(final String topic, boolean createIfMissing, CompletableFuture<Optional<Topic>> topicFuture, Map<String, String> properties) {
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    TopicName topicName = TopicName.get(topic);
    if (!pulsar.getNamespaceService().isServiceUnitActive(topicName)) {
        // namespace is being unloaded
        String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
        return;
    }
    if (isTransactionSystemTopic(topicName)) {
        String msg = String.format("Can not create transaction system topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new NotAllowedException(msg));
        return;
    }
    CompletableFuture<Void> maxTopicsCheck = createIfMissing ? checkMaxTopicsPerNamespace(topicName, 1) : CompletableFuture.completedFuture(null);
    maxTopicsCheck.thenCompose(__ -> getManagedLedgerConfig(topicName)).thenAccept(managedLedgerConfig -> {
        if (isBrokerEntryMetadataEnabled() || isBrokerPayloadProcessorEnabled()) {
            // init managedLedger interceptor
            Set<BrokerEntryMetadataInterceptor> interceptors = new HashSet<>();
            for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) {
                // add individual AppendOffsetMetadataInterceptor for each topic
                if (interceptor instanceof AppendIndexMetadataInterceptor) {
                    interceptors.add(new AppendIndexMetadataInterceptor());
                } else {
                    interceptors.add(interceptor);
                }
            }
            managedLedgerConfig.setManagedLedgerInterceptor(new ManagedLedgerInterceptorImpl(interceptors, brokerEntryPayloadProcessors));
        }
        managedLedgerConfig.setCreateIfMissing(createIfMissing);
        managedLedgerConfig.setProperties(properties);
        // Once we have the configuration, we can proceed with the async open operation
        managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                try {
                    PersistentTopic persistentTopic = isSystemTopic(topic) ? new SystemTopic(topic, ledger, BrokerService.this) : new PersistentTopic(topic, ledger, BrokerService.this);
                    CompletableFuture<Void> preCreateSubForCompaction = persistentTopic.preCreateSubscriptionForCompactionIfNeeded();
                    CompletableFuture<Void> replicationFuture = persistentTopic.initialize().thenCompose(__ -> persistentTopic.checkReplication());
                    CompletableFuture.allOf(preCreateSubForCompaction, replicationFuture).thenCompose(v -> {
                        // Also check dedup status
                        return persistentTopic.checkDeduplicationStatus();
                    }).thenRun(() -> {
                        log.info("Created topic {} - dedup is {}", topic, persistentTopic.isDeduplicationEnabled() ? "enabled" : "disabled");
                        long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
                        pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
                        if (topicFuture.isCompletedExceptionally()) {
                            log.warn("{} future is already completed with failure {}, closing the topic", topic, FutureUtil.getException(topicFuture));
                            persistentTopic.stopReplProducers().whenComplete((v, exception) -> {
                                topics.remove(topic, topicFuture);
                            });
                        } else {
                            addTopicToStatsMaps(topicName, persistentTopic);
                            topicFuture.complete(Optional.of(persistentTopic));
                        }
                    }).exceptionally((ex) -> {
                        log.warn("Replication or dedup check failed." + " Removing topic from topics list {}, {}", topic, ex);
                        persistentTopic.stopReplProducers().whenComplete((v, exception) -> {
                            topics.remove(topic, topicFuture);
                            topicFuture.completeExceptionally(ex);
                        });
                        return null;
                    });
                } catch (PulsarServerException e) {
                    log.warn("Failed to create topic {}-{}", topic, e.getMessage());
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(e);
                }
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                if (!createIfMissing && exception instanceof ManagedLedgerNotFoundException) {
                    // We were just trying to load a topic and the topic doesn't exist
                    topicFuture.complete(Optional.empty());
                } else {
                    log.warn("Failed to create topic {}", topic, exception);
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(new PersistenceException(exception));
                }
            }
        }, () -> isTopicNsOwnedByBroker(topicName), null);
    }).exceptionally((exception) -> {
        log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception);
        // remove topic from topics-map in different thread to avoid possible deadlock if
        // createPersistentTopic-thread only tries to handle this future-result
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(exception);
        return null;
    });
}
Also used : ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) Arrays(java.util.Arrays) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) LedgerOffloader(org.apache.bookkeeper.mledger.LedgerOffloader) BundlesQuotas(org.apache.pulsar.broker.cache.BundlesQuotas) LoadManager(org.apache.pulsar.broker.loadbalance.LoadManager) StringUtils(org.apache.commons.lang3.StringUtils) SystemTopic(org.apache.pulsar.broker.service.persistent.SystemTopic) FieldContext(org.apache.pulsar.common.configuration.FieldContext) Duration(java.time.Duration) Map(java.util.Map) BrokerInterceptor(org.apache.pulsar.broker.intercept.BrokerInterceptor) PersistentOfflineTopicStats(org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats) SocketChannel(io.netty.channel.socket.SocketChannel) ReadWriteLock(java.util.concurrent.locks.ReadWriteLock) LocalPoliciesResources(org.apache.pulsar.broker.resources.LocalPoliciesResources) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) CancellationException(java.util.concurrent.CancellationException) NettyFutureUtil(org.apache.pulsar.common.util.netty.NettyFutureUtil) PersistencePolicies(org.apache.pulsar.common.policies.data.PersistencePolicies) Set(java.util.Set) Futures(org.apache.bookkeeper.mledger.util.Futures) DelayedDeliveryTrackerLoader(org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerLoader) PublishRate(org.apache.pulsar.common.policies.data.PublishRate) Executors(java.util.concurrent.Executors) ConcurrentOpenHashSet(org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ClientBuilderImpl(org.apache.pulsar.client.impl.ClientBuilderImpl) ChannelFutures(org.apache.pulsar.common.util.netty.ChannelFutures) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) DispatchRateLimiter(org.apache.pulsar.broker.service.persistent.DispatchRateLimiter) ChannelOption(io.netty.channel.ChannelOption) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) AuthenticationService(org.apache.pulsar.broker.authentication.AuthenticationService) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) DelayedDeliveryTrackerFactory(org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory) RestException(org.apache.pulsar.common.util.RestException) ArrayList(java.util.ArrayList) TopicType(org.apache.pulsar.common.policies.data.TopicType) PulsarAdminBuilder(org.apache.pulsar.client.admin.PulsarAdminBuilder) PulsarService.isTransactionSystemTopic(org.apache.pulsar.broker.PulsarService.isTransactionSystemTopic) Lists(com.google.common.collect.Lists) AccessLevel(lombok.AccessLevel) BrokerEntryMetadataUtils(org.apache.pulsar.common.intercept.BrokerEntryMetadataUtils) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) PulsarClient(org.apache.pulsar.client.api.PulsarClient) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) PersistentDispatcherMultipleConsumers(org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers) ManagedLedgerNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException) Summary(org.apache.pulsar.broker.stats.prometheus.metrics.Summary) SslContext(io.netty.handler.ssl.SslContext) AutoSubscriptionCreationOverride(org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride) IOException(java.io.IOException) Field(java.lang.reflect.Field) PulsarService(org.apache.pulsar.broker.PulsarService) EventsTopicNames.checkTopicIsEventsNames(org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames) Channel(io.netty.channel.Channel) ExecutionException(java.util.concurrent.ExecutionException) SystemTopicClient(org.apache.pulsar.broker.systopic.SystemTopicClient) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback) Compactor(org.apache.pulsar.compaction.Compactor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) CollectionUtils.isEmpty(org.apache.commons.collections.CollectionUtils.isEmpty) AuthorizationService(org.apache.pulsar.broker.authorization.AuthorizationService) Metrics(org.apache.pulsar.common.stats.Metrics) DefaultThreadFactory(io.netty.util.concurrent.DefaultThreadFactory) ScheduledFuture(java.util.concurrent.ScheduledFuture) SocketAddress(java.net.SocketAddress) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) EntryFilterWithClassLoader(org.apache.pulsar.broker.service.plugin.EntryFilterWithClassLoader) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Notification(org.apache.pulsar.metadata.api.Notification) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) TransactionMetadataStoreService(org.apache.pulsar.broker.TransactionMetadataStoreService) EntryFilterProvider(org.apache.pulsar.broker.service.plugin.EntryFilterProvider) EventLoopUtil(org.apache.pulsar.common.util.netty.EventLoopUtil) ChannelInitializer(io.netty.channel.ChannelInitializer) OrderedScheduler(org.apache.bookkeeper.common.util.OrderedScheduler) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) OffloadPoliciesImpl(org.apache.pulsar.common.policies.data.OffloadPoliciesImpl) BindAddressValidator(org.apache.pulsar.broker.validator.BindAddressValidator) InetSocketAddress(java.net.InetSocketAddress) ObserverGauge(org.apache.pulsar.broker.stats.prometheus.metrics.ObserverGauge) PulsarByteBufAllocator(org.apache.pulsar.common.allocator.PulsarByteBufAllocator) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Queues(com.google.common.collect.Queues) List(java.util.List) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) ManagedLedgerPayloadProcessor(org.apache.pulsar.common.intercept.ManagedLedgerPayloadProcessor) FutureUtil(org.apache.pulsar.common.util.FutureUtil) Response(javax.ws.rs.core.Response) ClientConfigurationData(org.apache.pulsar.client.impl.conf.ClientConfigurationData) AppendIndexMetadataInterceptor(org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Optional(java.util.Optional) NamespaceResources(org.apache.pulsar.broker.resources.NamespaceResources) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) IsolatedBookieEnsemblePlacementPolicy(org.apache.pulsar.bookie.rackawareness.IsolatedBookieEnsemblePlacementPolicy) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) LongAdder(java.util.concurrent.atomic.LongAdder) Setter(lombok.Setter) TransactionMetadataStore(org.apache.pulsar.transaction.coordinator.TransactionMetadataStore) DeleteLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) RateLimiter(org.apache.pulsar.common.util.RateLimiter) AtomicReference(java.util.concurrent.atomic.AtomicReference) HashSet(java.util.HashSet) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) RejectedExecutionException(java.util.concurrent.RejectedExecutionException) ByteBuf(io.netty.buffer.ByteBuf) AutoTopicCreationOverride(org.apache.pulsar.common.policies.data.AutoTopicCreationOverride) BindAddress(org.apache.pulsar.common.configuration.BindAddress) ManagedLedgerInterceptorImpl(org.apache.pulsar.broker.intercept.ManagedLedgerInterceptorImpl) AdaptiveRecvByteBufAllocator(io.netty.channel.AdaptiveRecvByteBufAllocator) TopicDomain(org.apache.pulsar.common.naming.TopicDomain) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) FieldParser(org.apache.pulsar.common.util.FieldParser) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) EventLoopGroup(io.netty.channel.EventLoopGroup) Logger(org.slf4j.Logger) Semaphore(java.util.concurrent.Semaphore) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) SchemaVersion(org.apache.pulsar.common.protocol.schema.SchemaVersion) Maps(com.google.common.collect.Maps) OrderedExecutor(org.apache.bookkeeper.common.util.OrderedExecutor) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) Policies(org.apache.pulsar.common.policies.data.Policies) BrokerEntryMetadataInterceptor(org.apache.pulsar.common.intercept.BrokerEntryMetadataInterceptor) Closeable(java.io.Closeable) NonPersistentTopic(org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic) NotificationType(org.apache.pulsar.metadata.api.NotificationType) ServerBootstrap(io.netty.bootstrap.ServerBootstrap) VisibleForTesting(com.google.common.annotations.VisibleForTesting) AllArgsConstructor(lombok.AllArgsConstructor) Collections(java.util.Collections) BrokerEntryMetadataInterceptor(org.apache.pulsar.common.intercept.BrokerEntryMetadataInterceptor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) AppendIndexMetadataInterceptor(org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor) Set(java.util.Set) ConcurrentOpenHashSet(org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet) HashSet(java.util.HashSet) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) SystemTopic(org.apache.pulsar.broker.service.persistent.SystemTopic) PulsarService.isTransactionSystemTopic(org.apache.pulsar.broker.PulsarService.isTransactionSystemTopic) TopicName(org.apache.pulsar.common.naming.TopicName) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) ManagedLedgerNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException) ManagedLedgerInterceptorImpl(org.apache.pulsar.broker.intercept.ManagedLedgerInterceptorImpl) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) NonPersistentTopic(org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback)

Example 2 with SystemTopic

use of org.apache.pulsar.broker.service.persistent.SystemTopic in project pulsar by yahoo.

the class CompactionTest method testHealthCheckTopicNotCompacted.

@SneakyThrows
@Test
public void testHealthCheckTopicNotCompacted() {
    NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration());
    String topicV1 = "persistent://" + heartbeatNamespaceV1.toString() + "/healthcheck";
    NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), pulsar.getConfiguration());
    String topicV2 = heartbeatNamespaceV2.toString() + "/healthcheck";
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicV1).create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicV2).create();
    Optional<Topic> topicReferenceV1 = pulsar.getBrokerService().getTopic(topicV1, false).join();
    Optional<Topic> topicReferenceV2 = pulsar.getBrokerService().getTopic(topicV2, false).join();
    assertFalse(((SystemTopic) topicReferenceV1.get()).isCompactionEnabled());
    assertFalse(((SystemTopic) topicReferenceV2.get()).isCompactionEnabled());
    producer1.close();
    producer2.close();
}
Also used : NamespaceName(org.apache.pulsar.common.naming.NamespaceName) Topic(org.apache.pulsar.broker.service.Topic) SystemTopic(org.apache.pulsar.broker.service.persistent.SystemTopic) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) Test(org.testng.annotations.Test) MockedPulsarServiceBaseTest(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest) SneakyThrows(lombok.SneakyThrows)

Example 3 with SystemTopic

use of org.apache.pulsar.broker.service.persistent.SystemTopic in project incubator-pulsar by apache.

the class BrokerService method createPersistentTopic.

private void createPersistentTopic(final String topic, boolean createIfMissing, CompletableFuture<Optional<Topic>> topicFuture, Map<String, String> properties) {
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    TopicName topicName = TopicName.get(topic);
    if (!pulsar.getNamespaceService().isServiceUnitActive(topicName)) {
        // namespace is being unloaded
        String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
        return;
    }
    if (isTransactionInternalName(topicName)) {
        String msg = String.format("Can not create transaction system topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new NotAllowedException(msg));
        return;
    }
    CompletableFuture<Void> maxTopicsCheck = createIfMissing ? checkMaxTopicsPerNamespace(topicName, 1) : CompletableFuture.completedFuture(null);
    maxTopicsCheck.thenCompose(__ -> getManagedLedgerConfig(topicName)).thenAccept(managedLedgerConfig -> {
        if (isBrokerEntryMetadataEnabled() || isBrokerPayloadProcessorEnabled()) {
            // init managedLedger interceptor
            Set<BrokerEntryMetadataInterceptor> interceptors = new HashSet<>();
            for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) {
                // add individual AppendOffsetMetadataInterceptor for each topic
                if (interceptor instanceof AppendIndexMetadataInterceptor) {
                    interceptors.add(new AppendIndexMetadataInterceptor());
                } else {
                    interceptors.add(interceptor);
                }
            }
            managedLedgerConfig.setManagedLedgerInterceptor(new ManagedLedgerInterceptorImpl(interceptors, brokerEntryPayloadProcessors));
        }
        managedLedgerConfig.setCreateIfMissing(createIfMissing);
        managedLedgerConfig.setProperties(properties);
        // Once we have the configuration, we can proceed with the async open operation
        managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                try {
                    PersistentTopic persistentTopic = isSystemTopic(topic) ? new SystemTopic(topic, ledger, BrokerService.this) : new PersistentTopic(topic, ledger, BrokerService.this);
                    persistentTopic.initialize().thenCompose(__ -> persistentTopic.preCreateSubscriptionForCompactionIfNeeded()).thenCompose(__ -> persistentTopic.checkReplication()).thenCompose(v -> {
                        // Also check dedup status
                        return persistentTopic.checkDeduplicationStatus();
                    }).thenRun(() -> {
                        log.info("Created topic {} - dedup is {}", topic, persistentTopic.isDeduplicationEnabled() ? "enabled" : "disabled");
                        long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
                        pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
                        if (topicFuture.isCompletedExceptionally()) {
                            log.warn("{} future is already completed with failure {}, closing the" + " topic", topic, FutureUtil.getException(topicFuture));
                            persistentTopic.stopReplProducers().whenCompleteAsync((v, exception) -> {
                                topics.remove(topic, topicFuture);
                            }, executor());
                        } else {
                            addTopicToStatsMaps(topicName, persistentTopic);
                            topicFuture.complete(Optional.of(persistentTopic));
                        }
                    }).exceptionally((ex) -> {
                        log.warn("Replication or dedup check failed." + " Removing topic from topics list {}, {}", topic, ex);
                        persistentTopic.stopReplProducers().whenCompleteAsync((v, exception) -> {
                            topics.remove(topic, topicFuture);
                            topicFuture.completeExceptionally(ex);
                        }, executor());
                        return null;
                    });
                } catch (PulsarServerException e) {
                    log.warn("Failed to create topic {}-{}", topic, e.getMessage());
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(e);
                }
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                if (!createIfMissing && exception instanceof ManagedLedgerNotFoundException) {
                    // We were just trying to load a topic and the topic doesn't exist
                    topicFuture.complete(Optional.empty());
                } else {
                    log.warn("Failed to create topic {}", topic, exception);
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(new PersistenceException(exception));
                }
            }
        }, () -> isTopicNsOwnedByBroker(topicName), null);
    }).exceptionally((exception) -> {
        log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception);
        // remove topic from topics-map in different thread to avoid possible deadlock if
        // createPersistentTopic-thread only tries to handle this future-result
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(exception);
        return null;
    });
}
Also used : ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) Arrays(java.util.Arrays) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) LedgerOffloader(org.apache.bookkeeper.mledger.LedgerOffloader) BundlesQuotas(org.apache.pulsar.broker.cache.BundlesQuotas) LoadManager(org.apache.pulsar.broker.loadbalance.LoadManager) StringUtils(org.apache.commons.lang3.StringUtils) SystemTopic(org.apache.pulsar.broker.service.persistent.SystemTopic) FieldContext(org.apache.pulsar.common.configuration.FieldContext) Duration(java.time.Duration) Map(java.util.Map) BrokerInterceptor(org.apache.pulsar.broker.intercept.BrokerInterceptor) PersistentOfflineTopicStats(org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats) SocketChannel(io.netty.channel.socket.SocketChannel) ReadWriteLock(java.util.concurrent.locks.ReadWriteLock) LocalPoliciesResources(org.apache.pulsar.broker.resources.LocalPoliciesResources) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) CancellationException(java.util.concurrent.CancellationException) NettyFutureUtil(org.apache.pulsar.common.util.netty.NettyFutureUtil) PersistencePolicies(org.apache.pulsar.common.policies.data.PersistencePolicies) Set(java.util.Set) Futures(org.apache.bookkeeper.mledger.util.Futures) DelayedDeliveryTrackerLoader(org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerLoader) PublishRate(org.apache.pulsar.common.policies.data.PublishRate) Executors(java.util.concurrent.Executors) ConcurrentOpenHashSet(org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) SizeUnit(org.apache.pulsar.client.api.SizeUnit) ClientBuilderImpl(org.apache.pulsar.client.impl.ClientBuilderImpl) ChannelFutures(org.apache.pulsar.common.util.netty.ChannelFutures) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) DispatchRateLimiter(org.apache.pulsar.broker.service.persistent.DispatchRateLimiter) ChannelOption(io.netty.channel.ChannelOption) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) AuthenticationService(org.apache.pulsar.broker.authentication.AuthenticationService) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) NamespaceService(org.apache.pulsar.broker.namespace.NamespaceService) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) DelayedDeliveryTrackerFactory(org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory) RestException(org.apache.pulsar.common.util.RestException) ArrayList(java.util.ArrayList) TopicType(org.apache.pulsar.common.policies.data.TopicType) PulsarAdminBuilder(org.apache.pulsar.client.admin.PulsarAdminBuilder) Lists(com.google.common.collect.Lists) AccessLevel(lombok.AccessLevel) BrokerEntryMetadataUtils(org.apache.pulsar.common.intercept.BrokerEntryMetadataUtils) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) PropertiesUtils(org.apache.pulsar.client.internal.PropertiesUtils) PulsarClient(org.apache.pulsar.client.api.PulsarClient) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) PersistentDispatcherMultipleConsumers(org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers) NullLedgerOffloader(org.apache.bookkeeper.mledger.impl.NullLedgerOffloader) ManagedLedgerNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException) Summary(org.apache.pulsar.broker.stats.prometheus.metrics.Summary) SslContext(io.netty.handler.ssl.SslContext) AutoSubscriptionCreationOverride(org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride) IOException(java.io.IOException) Field(java.lang.reflect.Field) PulsarService(org.apache.pulsar.broker.PulsarService) Channel(io.netty.channel.Channel) ExecutionException(java.util.concurrent.ExecutionException) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback) Compactor(org.apache.pulsar.compaction.Compactor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) CollectionUtils.isEmpty(org.apache.commons.collections.CollectionUtils.isEmpty) AuthorizationService(org.apache.pulsar.broker.authorization.AuthorizationService) Metrics(org.apache.pulsar.common.stats.Metrics) DefaultThreadFactory(io.netty.util.concurrent.DefaultThreadFactory) ScheduledFuture(java.util.concurrent.ScheduledFuture) SocketAddress(java.net.SocketAddress) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) SystemTopicNames(org.apache.pulsar.common.naming.SystemTopicNames) EntryFilterWithClassLoader(org.apache.pulsar.broker.service.plugin.EntryFilterWithClassLoader) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Notification(org.apache.pulsar.metadata.api.Notification) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) TransactionMetadataStoreService(org.apache.pulsar.broker.TransactionMetadataStoreService) EntryFilterProvider(org.apache.pulsar.broker.service.plugin.EntryFilterProvider) EventLoopUtil(org.apache.pulsar.common.util.netty.EventLoopUtil) ChannelInitializer(io.netty.channel.ChannelInitializer) OrderedScheduler(org.apache.bookkeeper.common.util.OrderedScheduler) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) OffloadPoliciesImpl(org.apache.pulsar.common.policies.data.OffloadPoliciesImpl) LocalPolicies(org.apache.pulsar.common.policies.data.LocalPolicies) BindAddressValidator(org.apache.pulsar.broker.validator.BindAddressValidator) InetSocketAddress(java.net.InetSocketAddress) ObserverGauge(org.apache.pulsar.broker.stats.prometheus.metrics.ObserverGauge) PulsarByteBufAllocator(org.apache.pulsar.common.allocator.PulsarByteBufAllocator) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Queues(com.google.common.collect.Queues) List(java.util.List) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) ManagedLedgerPayloadProcessor(org.apache.pulsar.common.intercept.ManagedLedgerPayloadProcessor) FutureUtil(org.apache.pulsar.common.util.FutureUtil) Response(javax.ws.rs.core.Response) ClientConfigurationData(org.apache.pulsar.client.impl.conf.ClientConfigurationData) AppendIndexMetadataInterceptor(org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Optional(java.util.Optional) NamespaceResources(org.apache.pulsar.broker.resources.NamespaceResources) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) IsolatedBookieEnsemblePlacementPolicy(org.apache.pulsar.bookie.rackawareness.IsolatedBookieEnsemblePlacementPolicy) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) LongAdder(java.util.concurrent.atomic.LongAdder) Setter(lombok.Setter) TransactionMetadataStore(org.apache.pulsar.transaction.coordinator.TransactionMetadataStore) DeleteLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) RateLimiter(org.apache.pulsar.common.util.RateLimiter) AtomicReference(java.util.concurrent.atomic.AtomicReference) HashSet(java.util.HashSet) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) RejectedExecutionException(java.util.concurrent.RejectedExecutionException) ByteBuf(io.netty.buffer.ByteBuf) AutoTopicCreationOverride(org.apache.pulsar.common.policies.data.AutoTopicCreationOverride) BindAddress(org.apache.pulsar.common.configuration.BindAddress) ManagedLedgerInterceptorImpl(org.apache.pulsar.broker.intercept.ManagedLedgerInterceptorImpl) AdaptiveRecvByteBufAllocator(io.netty.channel.AdaptiveRecvByteBufAllocator) TopicDomain(org.apache.pulsar.common.naming.TopicDomain) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) FieldParser(org.apache.pulsar.common.util.FieldParser) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) EventLoopGroup(io.netty.channel.EventLoopGroup) Logger(org.slf4j.Logger) Semaphore(java.util.concurrent.Semaphore) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) Maps(com.google.common.collect.Maps) OrderedExecutor(org.apache.bookkeeper.common.util.OrderedExecutor) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) Policies(org.apache.pulsar.common.policies.data.Policies) BrokerEntryMetadataInterceptor(org.apache.pulsar.common.intercept.BrokerEntryMetadataInterceptor) Closeable(java.io.Closeable) NonPersistentTopic(org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic) NotificationType(org.apache.pulsar.metadata.api.NotificationType) ServerBootstrap(io.netty.bootstrap.ServerBootstrap) SystemTopicNames.isTransactionInternalName(org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName) VisibleForTesting(com.google.common.annotations.VisibleForTesting) AllArgsConstructor(lombok.AllArgsConstructor) Collections(java.util.Collections) BrokerEntryMetadataInterceptor(org.apache.pulsar.common.intercept.BrokerEntryMetadataInterceptor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) AppendIndexMetadataInterceptor(org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor) Set(java.util.Set) ConcurrentOpenHashSet(org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet) HashSet(java.util.HashSet) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) SystemTopic(org.apache.pulsar.broker.service.persistent.SystemTopic) TopicName(org.apache.pulsar.common.naming.TopicName) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) ManagedLedgerNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException) ManagedLedgerInterceptorImpl(org.apache.pulsar.broker.intercept.ManagedLedgerInterceptorImpl) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) NonPersistentTopic(org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback)

Example 4 with SystemTopic

use of org.apache.pulsar.broker.service.persistent.SystemTopic in project incubator-pulsar by apache.

the class CompactionTest method testHealthCheckTopicNotCompacted.

@SneakyThrows
@Test
public void testHealthCheckTopicNotCompacted() {
    NamespaceName heartbeatNamespaceV1 = NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration());
    String topicV1 = "persistent://" + heartbeatNamespaceV1.toString() + "/healthcheck";
    NamespaceName heartbeatNamespaceV2 = NamespaceService.getHeartbeatNamespaceV2(pulsar.getAdvertisedAddress(), pulsar.getConfiguration());
    String topicV2 = heartbeatNamespaceV2.toString() + "/healthcheck";
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicV1).create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicV2).create();
    Optional<Topic> topicReferenceV1 = pulsar.getBrokerService().getTopic(topicV1, false).join();
    Optional<Topic> topicReferenceV2 = pulsar.getBrokerService().getTopic(topicV2, false).join();
    assertFalse(((SystemTopic) topicReferenceV1.get()).isCompactionEnabled());
    assertFalse(((SystemTopic) topicReferenceV2.get()).isCompactionEnabled());
    producer1.close();
    producer2.close();
}
Also used : NamespaceName(org.apache.pulsar.common.naming.NamespaceName) Topic(org.apache.pulsar.broker.service.Topic) SystemTopic(org.apache.pulsar.broker.service.persistent.SystemTopic) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) Test(org.testng.annotations.Test) MockedPulsarServiceBaseTest(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest) SneakyThrows(lombok.SneakyThrows)

Example 5 with SystemTopic

use of org.apache.pulsar.broker.service.persistent.SystemTopic in project pulsar by yahoo.

the class BrokerService method createPersistentTopic.

private void createPersistentTopic(final String topic, boolean createIfMissing, CompletableFuture<Optional<Topic>> topicFuture, Map<String, String> properties) {
    final long topicCreateTimeMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime());
    TopicName topicName = TopicName.get(topic);
    if (!pulsar.getNamespaceService().isServiceUnitActive(topicName)) {
        // namespace is being unloaded
        String msg = String.format("Namespace is being unloaded, cannot add topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new ServiceUnitNotReadyException(msg));
        return;
    }
    if (isTransactionInternalName(topicName)) {
        String msg = String.format("Can not create transaction system topic %s", topic);
        log.warn(msg);
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(new NotAllowedException(msg));
        return;
    }
    CompletableFuture<Void> maxTopicsCheck = createIfMissing ? checkMaxTopicsPerNamespace(topicName, 1) : CompletableFuture.completedFuture(null);
    maxTopicsCheck.thenCompose(__ -> getManagedLedgerConfig(topicName)).thenAccept(managedLedgerConfig -> {
        if (isBrokerEntryMetadataEnabled() || isBrokerPayloadProcessorEnabled()) {
            // init managedLedger interceptor
            Set<BrokerEntryMetadataInterceptor> interceptors = new HashSet<>();
            for (BrokerEntryMetadataInterceptor interceptor : brokerEntryMetadataInterceptors) {
                // add individual AppendOffsetMetadataInterceptor for each topic
                if (interceptor instanceof AppendIndexMetadataInterceptor) {
                    interceptors.add(new AppendIndexMetadataInterceptor());
                } else {
                    interceptors.add(interceptor);
                }
            }
            managedLedgerConfig.setManagedLedgerInterceptor(new ManagedLedgerInterceptorImpl(interceptors, brokerEntryPayloadProcessors));
        }
        managedLedgerConfig.setCreateIfMissing(createIfMissing);
        managedLedgerConfig.setProperties(properties);
        // Once we have the configuration, we can proceed with the async open operation
        managedLedgerFactory.asyncOpen(topicName.getPersistenceNamingEncoding(), managedLedgerConfig, new OpenLedgerCallback() {

            @Override
            public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
                try {
                    PersistentTopic persistentTopic = isSystemTopic(topic) ? new SystemTopic(topic, ledger, BrokerService.this) : new PersistentTopic(topic, ledger, BrokerService.this);
                    persistentTopic.initialize().thenCompose(__ -> persistentTopic.preCreateSubscriptionForCompactionIfNeeded()).thenCompose(__ -> persistentTopic.checkReplication()).thenCompose(v -> {
                        // Also check dedup status
                        return persistentTopic.checkDeduplicationStatus();
                    }).thenRun(() -> {
                        log.info("Created topic {} - dedup is {}", topic, persistentTopic.isDeduplicationEnabled() ? "enabled" : "disabled");
                        long topicLoadLatencyMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime()) - topicCreateTimeMs;
                        pulsarStats.recordTopicLoadTimeValue(topic, topicLoadLatencyMs);
                        if (topicFuture.isCompletedExceptionally()) {
                            log.warn("{} future is already completed with failure {}, closing the" + " topic", topic, FutureUtil.getException(topicFuture));
                            persistentTopic.stopReplProducers().whenCompleteAsync((v, exception) -> {
                                topics.remove(topic, topicFuture);
                            }, executor());
                        } else {
                            addTopicToStatsMaps(topicName, persistentTopic);
                            topicFuture.complete(Optional.of(persistentTopic));
                        }
                    }).exceptionally((ex) -> {
                        log.warn("Replication or dedup check failed." + " Removing topic from topics list {}, {}", topic, ex);
                        persistentTopic.stopReplProducers().whenCompleteAsync((v, exception) -> {
                            topics.remove(topic, topicFuture);
                            topicFuture.completeExceptionally(ex);
                        }, executor());
                        return null;
                    });
                } catch (PulsarServerException e) {
                    log.warn("Failed to create topic {}-{}", topic, e.getMessage());
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(e);
                }
            }

            @Override
            public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
                if (!createIfMissing && exception instanceof ManagedLedgerNotFoundException) {
                    // We were just trying to load a topic and the topic doesn't exist
                    topicFuture.complete(Optional.empty());
                } else {
                    log.warn("Failed to create topic {}", topic, exception);
                    pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
                    topicFuture.completeExceptionally(new PersistenceException(exception));
                }
            }
        }, () -> isTopicNsOwnedByBroker(topicName), null);
    }).exceptionally((exception) -> {
        log.warn("[{}] Failed to get topic configuration: {}", topic, exception.getMessage(), exception);
        // remove topic from topics-map in different thread to avoid possible deadlock if
        // createPersistentTopic-thread only tries to handle this future-result
        pulsar.getExecutor().execute(() -> topics.remove(topic, topicFuture));
        topicFuture.completeExceptionally(exception);
        return null;
    });
}
Also used : ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) Arrays(java.util.Arrays) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) LedgerOffloader(org.apache.bookkeeper.mledger.LedgerOffloader) BundlesQuotas(org.apache.pulsar.broker.cache.BundlesQuotas) LoadManager(org.apache.pulsar.broker.loadbalance.LoadManager) StringUtils(org.apache.commons.lang3.StringUtils) SystemTopic(org.apache.pulsar.broker.service.persistent.SystemTopic) FieldContext(org.apache.pulsar.common.configuration.FieldContext) Duration(java.time.Duration) Map(java.util.Map) BrokerInterceptor(org.apache.pulsar.broker.intercept.BrokerInterceptor) PersistentOfflineTopicStats(org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats) SocketChannel(io.netty.channel.socket.SocketChannel) ReadWriteLock(java.util.concurrent.locks.ReadWriteLock) LocalPoliciesResources(org.apache.pulsar.broker.resources.LocalPoliciesResources) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) CancellationException(java.util.concurrent.CancellationException) NettyFutureUtil(org.apache.pulsar.common.util.netty.NettyFutureUtil) PersistencePolicies(org.apache.pulsar.common.policies.data.PersistencePolicies) Set(java.util.Set) Futures(org.apache.bookkeeper.mledger.util.Futures) DelayedDeliveryTrackerLoader(org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerLoader) PublishRate(org.apache.pulsar.common.policies.data.PublishRate) Executors(java.util.concurrent.Executors) ConcurrentOpenHashSet(org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet) StringUtils.isNotBlank(org.apache.commons.lang3.StringUtils.isNotBlank) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) SizeUnit(org.apache.pulsar.client.api.SizeUnit) ClientBuilderImpl(org.apache.pulsar.client.impl.ClientBuilderImpl) ChannelFutures(org.apache.pulsar.common.util.netty.ChannelFutures) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) DispatchRateLimiter(org.apache.pulsar.broker.service.persistent.DispatchRateLimiter) ChannelOption(io.netty.channel.ChannelOption) ManagedLedgerFactory(org.apache.bookkeeper.mledger.ManagedLedgerFactory) AuthenticationService(org.apache.pulsar.broker.authentication.AuthenticationService) ReentrantReadWriteLock(java.util.concurrent.locks.ReentrantReadWriteLock) NamespaceService(org.apache.pulsar.broker.namespace.NamespaceService) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) DelayedDeliveryTrackerFactory(org.apache.pulsar.broker.delayed.DelayedDeliveryTrackerFactory) RestException(org.apache.pulsar.common.util.RestException) ArrayList(java.util.ArrayList) TopicType(org.apache.pulsar.common.policies.data.TopicType) PulsarAdminBuilder(org.apache.pulsar.client.admin.PulsarAdminBuilder) Lists(com.google.common.collect.Lists) AccessLevel(lombok.AccessLevel) BrokerEntryMetadataUtils(org.apache.pulsar.common.intercept.BrokerEntryMetadataUtils) ScheduledExecutorService(java.util.concurrent.ScheduledExecutorService) PulsarClient(org.apache.pulsar.client.api.PulsarClient) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) PersistentDispatcherMultipleConsumers(org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers) NullLedgerOffloader(org.apache.bookkeeper.mledger.impl.NullLedgerOffloader) ManagedLedgerNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException) Summary(org.apache.pulsar.broker.stats.prometheus.metrics.Summary) SslContext(io.netty.handler.ssl.SslContext) AutoSubscriptionCreationOverride(org.apache.pulsar.common.policies.data.AutoSubscriptionCreationOverride) IOException(java.io.IOException) Field(java.lang.reflect.Field) PulsarService(org.apache.pulsar.broker.PulsarService) Channel(io.netty.channel.Channel) ExecutionException(java.util.concurrent.ExecutionException) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback) Compactor(org.apache.pulsar.compaction.Compactor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) CollectionUtils.isEmpty(org.apache.commons.collections.CollectionUtils.isEmpty) AuthorizationService(org.apache.pulsar.broker.authorization.AuthorizationService) Metrics(org.apache.pulsar.common.stats.Metrics) DefaultThreadFactory(io.netty.util.concurrent.DefaultThreadFactory) ScheduledFuture(java.util.concurrent.ScheduledFuture) SocketAddress(java.net.SocketAddress) LoggerFactory(org.slf4j.LoggerFactory) TimeoutException(java.util.concurrent.TimeoutException) ClusterData(org.apache.pulsar.common.policies.data.ClusterData) SystemTopicNames(org.apache.pulsar.common.naming.SystemTopicNames) EntryFilterWithClassLoader(org.apache.pulsar.broker.service.plugin.EntryFilterWithClassLoader) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) Notification(org.apache.pulsar.metadata.api.Notification) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) TransactionMetadataStoreService(org.apache.pulsar.broker.TransactionMetadataStoreService) EntryFilterProvider(org.apache.pulsar.broker.service.plugin.EntryFilterProvider) EventLoopUtil(org.apache.pulsar.common.util.netty.EventLoopUtil) ChannelInitializer(io.netty.channel.ChannelInitializer) OrderedScheduler(org.apache.bookkeeper.common.util.OrderedScheduler) ImmutableMap(com.google.common.collect.ImmutableMap) Predicate(java.util.function.Predicate) OffloadPoliciesImpl(org.apache.pulsar.common.policies.data.OffloadPoliciesImpl) LocalPolicies(org.apache.pulsar.common.policies.data.LocalPolicies) BindAddressValidator(org.apache.pulsar.broker.validator.BindAddressValidator) InetSocketAddress(java.net.InetSocketAddress) ObserverGauge(org.apache.pulsar.broker.stats.prometheus.metrics.ObserverGauge) PulsarByteBufAllocator(org.apache.pulsar.common.allocator.PulsarByteBufAllocator) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) Queues(com.google.common.collect.Queues) List(java.util.List) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) ManagedLedgerPayloadProcessor(org.apache.pulsar.common.intercept.ManagedLedgerPayloadProcessor) FutureUtil(org.apache.pulsar.common.util.FutureUtil) Response(javax.ws.rs.core.Response) ClientConfigurationData(org.apache.pulsar.client.impl.conf.ClientConfigurationData) AppendIndexMetadataInterceptor(org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor) ClientBuilder(org.apache.pulsar.client.api.ClientBuilder) Optional(java.util.Optional) NamespaceResources(org.apache.pulsar.broker.resources.NamespaceResources) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) IsolatedBookieEnsemblePlacementPolicy(org.apache.pulsar.bookie.rackawareness.IsolatedBookieEnsemblePlacementPolicy) ConcurrentLinkedQueue(java.util.concurrent.ConcurrentLinkedQueue) LongAdder(java.util.concurrent.atomic.LongAdder) Setter(lombok.Setter) TransactionMetadataStore(org.apache.pulsar.transaction.coordinator.TransactionMetadataStore) DeleteLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) RateLimiter(org.apache.pulsar.common.util.RateLimiter) AtomicReference(java.util.concurrent.atomic.AtomicReference) HashSet(java.util.HashSet) SafeRun.safeRun(org.apache.bookkeeper.mledger.util.SafeRun.safeRun) RejectedExecutionException(java.util.concurrent.RejectedExecutionException) ByteBuf(io.netty.buffer.ByteBuf) AutoTopicCreationOverride(org.apache.pulsar.common.policies.data.AutoTopicCreationOverride) BindAddress(org.apache.pulsar.common.configuration.BindAddress) ManagedLedgerInterceptorImpl(org.apache.pulsar.broker.intercept.ManagedLedgerInterceptorImpl) AdaptiveRecvByteBufAllocator(io.netty.channel.AdaptiveRecvByteBufAllocator) TopicDomain(org.apache.pulsar.common.naming.TopicDomain) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) FieldParser(org.apache.pulsar.common.util.FieldParser) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) EventLoopGroup(io.netty.channel.EventLoopGroup) Logger(org.slf4j.Logger) Semaphore(java.util.concurrent.Semaphore) ServiceConfiguration(org.apache.pulsar.broker.ServiceConfiguration) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) Maps(com.google.common.collect.Maps) OrderedExecutor(org.apache.bookkeeper.common.util.OrderedExecutor) TimeUnit(java.util.concurrent.TimeUnit) Consumer(java.util.function.Consumer) Policies(org.apache.pulsar.common.policies.data.Policies) BrokerEntryMetadataInterceptor(org.apache.pulsar.common.intercept.BrokerEntryMetadataInterceptor) Closeable(java.io.Closeable) NonPersistentTopic(org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic) NotificationType(org.apache.pulsar.metadata.api.NotificationType) ServerBootstrap(io.netty.bootstrap.ServerBootstrap) SystemTopicNames.isTransactionInternalName(org.apache.pulsar.common.naming.SystemTopicNames.isTransactionInternalName) VisibleForTesting(com.google.common.annotations.VisibleForTesting) AllArgsConstructor(lombok.AllArgsConstructor) Collections(java.util.Collections) BrokerEntryMetadataInterceptor(org.apache.pulsar.common.intercept.BrokerEntryMetadataInterceptor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) AppendIndexMetadataInterceptor(org.apache.pulsar.common.intercept.AppendIndexMetadataInterceptor) Set(java.util.Set) ConcurrentOpenHashSet(org.apache.pulsar.common.util.collections.ConcurrentOpenHashSet) HashSet(java.util.HashSet) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) SystemTopic(org.apache.pulsar.broker.service.persistent.SystemTopic) TopicName(org.apache.pulsar.common.naming.TopicName) ServiceUnitNotReadyException(org.apache.pulsar.broker.service.BrokerServiceException.ServiceUnitNotReadyException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) ManagedLedgerNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException) ManagedLedgerInterceptorImpl(org.apache.pulsar.broker.intercept.ManagedLedgerInterceptorImpl) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) NonPersistentTopic(org.apache.pulsar.broker.service.nonpersistent.NonPersistentTopic) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) OpenLedgerCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback)

Aggregations

PersistentTopic (org.apache.pulsar.broker.service.persistent.PersistentTopic)5 SystemTopic (org.apache.pulsar.broker.service.persistent.SystemTopic)5 NamespaceName (org.apache.pulsar.common.naming.NamespaceName)5 VisibleForTesting (com.google.common.annotations.VisibleForTesting)3 Preconditions.checkArgument (com.google.common.base.Preconditions.checkArgument)3 Preconditions.checkNotNull (com.google.common.base.Preconditions.checkNotNull)3 ImmutableMap (com.google.common.collect.ImmutableMap)3 Lists (com.google.common.collect.Lists)3 Maps (com.google.common.collect.Maps)3 Queues (com.google.common.collect.Queues)3 ServerBootstrap (io.netty.bootstrap.ServerBootstrap)3 ByteBuf (io.netty.buffer.ByteBuf)3 AdaptiveRecvByteBufAllocator (io.netty.channel.AdaptiveRecvByteBufAllocator)3 Channel (io.netty.channel.Channel)3 ChannelInitializer (io.netty.channel.ChannelInitializer)3 ChannelOption (io.netty.channel.ChannelOption)3 EventLoopGroup (io.netty.channel.EventLoopGroup)3 SocketChannel (io.netty.channel.socket.SocketChannel)3 SslContext (io.netty.handler.ssl.SslContext)3 DefaultThreadFactory (io.netty.util.concurrent.DefaultThreadFactory)3