Search in sources :

Example 1 with BookkeeperSchemaStorage

use of org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage in project pulsar by apache.

the class PersistentTopic method getInternalStats.

@Override
public CompletableFuture<PersistentTopicInternalStats> getInternalStats(boolean includeLedgerMetadata) {
    CompletableFuture<PersistentTopicInternalStats> statFuture = new CompletableFuture<>();
    PersistentTopicInternalStats stats = new PersistentTopicInternalStats();
    ManagedLedgerImpl ml = (ManagedLedgerImpl) ledger;
    stats.entriesAddedCounter = ml.getEntriesAddedCounter();
    stats.numberOfEntries = ml.getNumberOfEntries();
    stats.totalSize = ml.getTotalSize();
    stats.currentLedgerEntries = ml.getCurrentLedgerEntries();
    stats.currentLedgerSize = ml.getCurrentLedgerSize();
    stats.lastLedgerCreatedTimestamp = DateFormatter.format(ml.getLastLedgerCreatedTimestamp());
    if (ml.getLastLedgerCreationFailureTimestamp() != 0) {
        stats.lastLedgerCreationFailureTimestamp = DateFormatter.format(ml.getLastLedgerCreationFailureTimestamp());
    }
    stats.waitingCursorsCount = ml.getWaitingCursorsCount();
    stats.pendingAddEntriesCount = ml.getPendingAddEntriesCount();
    stats.lastConfirmedEntry = ml.getLastConfirmedEntry().toString();
    stats.state = ml.getState().toString();
    stats.ledgers = Lists.newArrayList();
    List<CompletableFuture<String>> futures = Lists.newArrayList();
    CompletableFuture<Set<String>> availableBookiesFuture = brokerService.pulsar().getPulsarResources().getBookieResources().listAvailableBookiesAsync();
    futures.add(availableBookiesFuture.handle((strings, throwable) -> null));
    availableBookiesFuture.whenComplete((bookies, e) -> {
        if (e != null) {
            log.error("[{}] Failed to fetch available bookies.", topic, e);
            statFuture.completeExceptionally(e);
        } else {
            ml.getLedgersInfo().forEach((id, li) -> {
                LedgerInfo info = new LedgerInfo();
                info.ledgerId = li.getLedgerId();
                info.entries = li.getEntries();
                info.size = li.getSize();
                info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete();
                stats.ledgers.add(info);
                if (includeLedgerMetadata) {
                    futures.add(ml.getLedgerMetadata(li.getLedgerId()).handle((lMetadata, ex) -> {
                        if (ex == null) {
                            info.metadata = lMetadata;
                        }
                        return null;
                    }));
                    futures.add(ml.getEnsemblesAsync(li.getLedgerId()).handle((ensembles, ex) -> {
                        if (ex == null) {
                            info.underReplicated = !bookies.containsAll(ensembles.stream().map(BookieId::toString).collect(Collectors.toList()));
                        }
                        return null;
                    }));
                }
            });
        }
    });
    // Add ledger info for compacted topic ledger if exist.
    LedgerInfo info = new LedgerInfo();
    info.ledgerId = -1;
    info.entries = -1;
    info.size = -1;
    Optional<CompactedTopicContext> compactedTopicContext = getCompactedTopicContext();
    if (compactedTopicContext.isPresent()) {
        CompactedTopicContext ledgerContext = compactedTopicContext.get();
        info.ledgerId = ledgerContext.getLedger().getId();
        info.entries = ledgerContext.getLedger().getLastAddConfirmed() + 1;
        info.size = ledgerContext.getLedger().getLength();
    }
    stats.compactedLedger = info;
    stats.cursors = Maps.newTreeMap();
    ml.getCursors().forEach(c -> {
        ManagedCursorImpl cursor = (ManagedCursorImpl) c;
        CursorStats cs = new CursorStats();
        cs.markDeletePosition = cursor.getMarkDeletedPosition().toString();
        cs.readPosition = cursor.getReadPosition().toString();
        cs.waitingReadOp = cursor.hasPendingReadRequest();
        cs.pendingReadOps = cursor.getPendingReadOpsCount();
        cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter();
        cs.cursorLedger = cursor.getCursorLedger();
        cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry();
        cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages();
        cs.lastLedgerSwitchTimestamp = DateFormatter.format(cursor.getLastLedgerSwitchTimestamp());
        cs.state = cursor.getState();
        cs.numberOfEntriesSinceFirstNotAckedMessage = cursor.getNumberOfEntriesSinceFirstNotAckedMessage();
        cs.totalNonContiguousDeletedMessagesRange = cursor.getTotalNonContiguousDeletedMessagesRange();
        cs.properties = cursor.getProperties();
        // subscription metrics
        PersistentSubscription sub = subscriptions.get(Codec.decode(c.getName()));
        if (sub != null) {
            if (sub.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) {
                PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) sub.getDispatcher();
                cs.subscriptionHavePendingRead = dispatcher.havePendingRead;
                cs.subscriptionHavePendingReplayRead = dispatcher.havePendingReplayRead;
            } else if (sub.getDispatcher() instanceof PersistentDispatcherSingleActiveConsumer) {
                PersistentDispatcherSingleActiveConsumer dispatcher = (PersistentDispatcherSingleActiveConsumer) sub.getDispatcher();
                cs.subscriptionHavePendingRead = dispatcher.havePendingRead;
            }
        }
        stats.cursors.put(cursor.getName(), cs);
    });
    // Schema store ledgers
    String schemaId;
    try {
        schemaId = TopicName.get(topic).getSchemaName();
    } catch (Throwable t) {
        statFuture.completeExceptionally(t);
        return statFuture;
    }
    CompletableFuture<Void> schemaStoreLedgersFuture = new CompletableFuture<>();
    stats.schemaLedgers = Collections.synchronizedList(new ArrayList<>());
    if (brokerService.getPulsar().getSchemaStorage() != null && brokerService.getPulsar().getSchemaStorage() instanceof BookkeeperSchemaStorage) {
        ((BookkeeperSchemaStorage) brokerService.getPulsar().getSchemaStorage()).getStoreLedgerIdsBySchemaId(schemaId).thenAccept(ledgers -> {
            List<CompletableFuture<Void>> getLedgerMetadataFutures = new ArrayList<>();
            ledgers.forEach(ledgerId -> {
                CompletableFuture<Void> completableFuture = new CompletableFuture<>();
                getLedgerMetadataFutures.add(completableFuture);
                CompletableFuture<LedgerMetadata> metadataFuture = null;
                try {
                    metadataFuture = brokerService.getPulsar().getBookKeeperClient().getLedgerMetadata(ledgerId);
                } catch (NullPointerException e) {
                    // related to bookkeeper issue https://github.com/apache/bookkeeper/issues/2741
                    if (log.isDebugEnabled()) {
                        log.debug("{{}} Failed to get ledger metadata for the schema ledger {}", topic, ledgerId, e);
                    }
                }
                if (metadataFuture != null) {
                    metadataFuture.thenAccept(metadata -> {
                        LedgerInfo schemaLedgerInfo = new LedgerInfo();
                        schemaLedgerInfo.ledgerId = metadata.getLedgerId();
                        schemaLedgerInfo.entries = metadata.getLastEntryId() + 1;
                        schemaLedgerInfo.size = metadata.getLength();
                        if (includeLedgerMetadata) {
                            info.metadata = metadata.toSafeString();
                        }
                        stats.schemaLedgers.add(schemaLedgerInfo);
                        completableFuture.complete(null);
                    }).exceptionally(e -> {
                        completableFuture.completeExceptionally(e);
                        return null;
                    });
                } else {
                    completableFuture.complete(null);
                }
            });
            FutureUtil.waitForAll(getLedgerMetadataFutures).thenRun(() -> {
                schemaStoreLedgersFuture.complete(null);
            }).exceptionally(e -> {
                schemaStoreLedgersFuture.completeExceptionally(e);
                return null;
            });
        }).exceptionally(e -> {
            schemaStoreLedgersFuture.completeExceptionally(e);
            return null;
        });
    } else {
        schemaStoreLedgersFuture.complete(null);
    }
    schemaStoreLedgersFuture.thenRun(() -> {
        if (futures != null) {
            FutureUtil.waitForAll(futures).handle((res, ex) -> {
                statFuture.complete(stats);
                return null;
            });
        } else {
            statFuture.complete(stats);
        }
    }).exceptionally(e -> {
        statFuture.completeExceptionally(e);
        return null;
    });
    return statFuture;
}
Also used : EventsTopicNames(org.apache.pulsar.common.events.EventsTopicNames) SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) Topic(org.apache.pulsar.broker.service.Topic) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) StringUtils(org.apache.commons.lang3.StringUtils) MetadataNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) TxnID(org.apache.pulsar.client.api.transaction.TxnID) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) AlreadyRunningException(org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException) Map(java.util.Map) StatsOutputStream(org.apache.pulsar.utils.StatsOutputStream) EnumSet(java.util.EnumSet) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) UpdatePropertiesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.UpdatePropertiesCallback) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) ManagedLedgerAlreadyClosedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException) CancellationException(java.util.concurrent.CancellationException) ManagedCursorContainer(org.apache.bookkeeper.mledger.impl.ManagedCursorContainer) Set(java.util.Set) BrokerService(org.apache.pulsar.broker.service.BrokerService) NamespaceStats(org.apache.pulsar.broker.stats.NamespaceStats) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) OffloadProcessStatus(org.apache.pulsar.client.admin.OffloadProcessStatus) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) TransactionPendingAckStats(org.apache.pulsar.common.policies.data.TransactionPendingAckStats) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) TopicTerminatedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) ManagedLedgerTerminatedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) BookieId(org.apache.bookkeeper.net.BookieId) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) TransactionInPendingAckStats(org.apache.pulsar.common.policies.data.TransactionInPendingAckStats) Subscription(org.apache.pulsar.broker.service.Subscription) LedgerInfo(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo) Consumer(org.apache.pulsar.broker.service.Consumer) TerminateCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback) ArrayList(java.util.ArrayList) TopicClosedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException) Commands(org.apache.pulsar.common.protocol.Commands) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Lists(com.google.common.collect.Lists) CursorStats(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats) DateFormatter(org.apache.pulsar.common.util.DateFormatter) TopicBacklogQuotaExceededException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBacklogQuotaExceededException) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) CompactorMXBean(org.apache.pulsar.compaction.CompactorMXBean) Dispatcher(org.apache.pulsar.broker.service.Dispatcher) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) BrokerServiceException(org.apache.pulsar.broker.service.BrokerServiceException) StreamingStats(org.apache.pulsar.broker.service.StreamingStats) AbstractReplicator(org.apache.pulsar.broker.service.AbstractReplicator) TopicFencedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException) EventsTopicNames.checkTopicIsEventsNames(org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsEventsNames) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Compactor(org.apache.pulsar.compaction.Compactor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) Codec(org.apache.pulsar.common.util.Codec) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) ScheduledFuture(java.util.concurrent.ScheduledFuture) COMPACTION_SUBSCRIPTION(org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION) BookkeeperSchemaStorage(org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage) BiFunction(java.util.function.BiFunction) UnsupportedVersionException(org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) LoggerFactory(org.slf4j.LoggerFactory) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) Type(org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type) BacklogQuotaType(org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) TxnAction(org.apache.pulsar.common.api.proto.TxnAction) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) NamespaceName(org.apache.pulsar.common.naming.NamespaceName) TransactionInBufferStats(org.apache.pulsar.common.policies.data.TransactionInBufferStats) ReplicationMetrics(org.apache.pulsar.broker.stats.ReplicationMetrics) CompletionException(java.util.concurrent.CompletionException) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) CompactedTopicContext(org.apache.pulsar.compaction.CompactedTopicContext) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) InactiveTopicDeleteMode(org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) MLPendingAckStore(org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore) Optional(java.util.Optional) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) LedgerMetadata(org.apache.bookkeeper.client.api.LedgerMetadata) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) LongAdder(java.util.concurrent.atomic.LongAdder) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) CommandSubscribe(org.apache.pulsar.common.api.proto.CommandSubscribe) SubscriptionOption(org.apache.pulsar.broker.service.SubscriptionOption) ByteBuf(io.netty.buffer.ByteBuf) CompactedTopicImpl(org.apache.pulsar.compaction.CompactedTopicImpl) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) TransactionBufferDisable(org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable) LongRunningProcessStatus(org.apache.pulsar.client.admin.LongRunningProcessStatus) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) AbstractTopic(org.apache.pulsar.broker.service.AbstractTopic) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) MapUtils(org.apache.commons.collections4.MapUtils) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) TransportCnx(org.apache.pulsar.broker.service.TransportCnx) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) Logger(org.slf4j.Logger) CompactedTopic(org.apache.pulsar.compaction.CompactedTopic) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) OffloadCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OffloadCallback) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) Producer(org.apache.pulsar.broker.service.Producer) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) Maps(com.google.common.collect.Maps) TransactionBufferStats(org.apache.pulsar.common.policies.data.TransactionBufferStats) TransactionBuffer(org.apache.pulsar.broker.transaction.buffer.TransactionBuffer) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) MessageId(org.apache.pulsar.client.api.MessageId) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) Clock(java.time.Clock) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) Replicator(org.apache.pulsar.broker.service.Replicator) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) EnumSet(java.util.EnumSet) Set(java.util.Set) BookkeeperSchemaStorage(org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage) ArrayList(java.util.ArrayList) CompletableFuture(java.util.concurrent.CompletableFuture) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) ArrayList(java.util.ArrayList) List(java.util.List) CompactedTopicContext(org.apache.pulsar.compaction.CompactedTopicContext) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) LedgerInfo(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) CursorStats(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats)

Example 2 with BookkeeperSchemaStorage

use of org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage in project incubator-pulsar by apache.

the class PersistentTopic method getInternalStats.

@Override
public CompletableFuture<PersistentTopicInternalStats> getInternalStats(boolean includeLedgerMetadata) {
    CompletableFuture<PersistentTopicInternalStats> statFuture = new CompletableFuture<>();
    PersistentTopicInternalStats stats = new PersistentTopicInternalStats();
    ManagedLedgerImpl ml = (ManagedLedgerImpl) ledger;
    stats.entriesAddedCounter = ml.getEntriesAddedCounter();
    stats.numberOfEntries = ml.getNumberOfEntries();
    stats.totalSize = ml.getTotalSize();
    stats.currentLedgerEntries = ml.getCurrentLedgerEntries();
    stats.currentLedgerSize = ml.getCurrentLedgerSize();
    stats.lastLedgerCreatedTimestamp = DateFormatter.format(ml.getLastLedgerCreatedTimestamp());
    if (ml.getLastLedgerCreationFailureTimestamp() != 0) {
        stats.lastLedgerCreationFailureTimestamp = DateFormatter.format(ml.getLastLedgerCreationFailureTimestamp());
    }
    stats.waitingCursorsCount = ml.getWaitingCursorsCount();
    stats.pendingAddEntriesCount = ml.getPendingAddEntriesCount();
    stats.lastConfirmedEntry = ml.getLastConfirmedEntry().toString();
    stats.state = ml.getState().toString();
    stats.ledgers = Lists.newArrayList();
    List<CompletableFuture<String>> futures = Lists.newArrayList();
    CompletableFuture<Set<String>> availableBookiesFuture = brokerService.pulsar().getPulsarResources().getBookieResources().listAvailableBookiesAsync();
    futures.add(availableBookiesFuture.handle((strings, throwable) -> null));
    availableBookiesFuture.whenComplete((bookies, e) -> {
        if (e != null) {
            log.error("[{}] Failed to fetch available bookies.", topic, e);
            statFuture.completeExceptionally(e);
        } else {
            ml.getLedgersInfo().forEach((id, li) -> {
                LedgerInfo info = new LedgerInfo();
                info.ledgerId = li.getLedgerId();
                info.entries = li.getEntries();
                info.size = li.getSize();
                info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete();
                stats.ledgers.add(info);
                if (includeLedgerMetadata) {
                    futures.add(ml.getLedgerMetadata(li.getLedgerId()).handle((lMetadata, ex) -> {
                        if (ex == null) {
                            info.metadata = lMetadata;
                        }
                        return null;
                    }));
                    futures.add(ml.getEnsemblesAsync(li.getLedgerId()).handle((ensembles, ex) -> {
                        if (ex == null) {
                            info.underReplicated = !bookies.containsAll(ensembles.stream().map(BookieId::toString).collect(Collectors.toList()));
                        }
                        return null;
                    }));
                }
            });
        }
    });
    // Add ledger info for compacted topic ledger if exist.
    LedgerInfo info = new LedgerInfo();
    info.ledgerId = -1;
    info.entries = -1;
    info.size = -1;
    Optional<CompactedTopicContext> compactedTopicContext = getCompactedTopicContext();
    if (compactedTopicContext.isPresent()) {
        CompactedTopicContext ledgerContext = compactedTopicContext.get();
        info.ledgerId = ledgerContext.getLedger().getId();
        info.entries = ledgerContext.getLedger().getLastAddConfirmed() + 1;
        info.size = ledgerContext.getLedger().getLength();
    }
    stats.compactedLedger = info;
    stats.cursors = Maps.newTreeMap();
    ml.getCursors().forEach(c -> {
        ManagedCursorImpl cursor = (ManagedCursorImpl) c;
        CursorStats cs = new CursorStats();
        cs.markDeletePosition = cursor.getMarkDeletedPosition().toString();
        cs.readPosition = cursor.getReadPosition().toString();
        cs.waitingReadOp = cursor.hasPendingReadRequest();
        cs.pendingReadOps = cursor.getPendingReadOpsCount();
        cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter();
        cs.cursorLedger = cursor.getCursorLedger();
        cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry();
        cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages();
        cs.lastLedgerSwitchTimestamp = DateFormatter.format(cursor.getLastLedgerSwitchTimestamp());
        cs.state = cursor.getState();
        cs.numberOfEntriesSinceFirstNotAckedMessage = cursor.getNumberOfEntriesSinceFirstNotAckedMessage();
        cs.totalNonContiguousDeletedMessagesRange = cursor.getTotalNonContiguousDeletedMessagesRange();
        cs.properties = cursor.getProperties();
        // subscription metrics
        PersistentSubscription sub = subscriptions.get(Codec.decode(c.getName()));
        if (sub != null) {
            if (sub.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) {
                PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) sub.getDispatcher();
                cs.subscriptionHavePendingRead = dispatcher.havePendingRead;
                cs.subscriptionHavePendingReplayRead = dispatcher.havePendingReplayRead;
            } else if (sub.getDispatcher() instanceof PersistentDispatcherSingleActiveConsumer) {
                PersistentDispatcherSingleActiveConsumer dispatcher = (PersistentDispatcherSingleActiveConsumer) sub.getDispatcher();
                cs.subscriptionHavePendingRead = dispatcher.havePendingRead;
            }
        }
        stats.cursors.put(cursor.getName(), cs);
    });
    // Schema store ledgers
    String schemaId;
    try {
        schemaId = TopicName.get(topic).getSchemaName();
    } catch (Throwable t) {
        statFuture.completeExceptionally(t);
        return statFuture;
    }
    CompletableFuture<Void> schemaStoreLedgersFuture = new CompletableFuture<>();
    stats.schemaLedgers = Collections.synchronizedList(new ArrayList<>());
    if (brokerService.getPulsar().getSchemaStorage() != null && brokerService.getPulsar().getSchemaStorage() instanceof BookkeeperSchemaStorage) {
        ((BookkeeperSchemaStorage) brokerService.getPulsar().getSchemaStorage()).getStoreLedgerIdsBySchemaId(schemaId).thenAccept(ledgers -> {
            List<CompletableFuture<Void>> getLedgerMetadataFutures = new ArrayList<>();
            ledgers.forEach(ledgerId -> {
                CompletableFuture<Void> completableFuture = new CompletableFuture<>();
                getLedgerMetadataFutures.add(completableFuture);
                CompletableFuture<LedgerMetadata> metadataFuture = null;
                try {
                    metadataFuture = brokerService.getPulsar().getBookKeeperClient().getLedgerMetadata(ledgerId);
                } catch (NullPointerException e) {
                    // related to bookkeeper issue https://github.com/apache/bookkeeper/issues/2741
                    if (log.isDebugEnabled()) {
                        log.debug("{{}} Failed to get ledger metadata for the schema ledger {}", topic, ledgerId, e);
                    }
                }
                if (metadataFuture != null) {
                    metadataFuture.thenAccept(metadata -> {
                        LedgerInfo schemaLedgerInfo = new LedgerInfo();
                        schemaLedgerInfo.ledgerId = metadata.getLedgerId();
                        schemaLedgerInfo.entries = metadata.getLastEntryId() + 1;
                        schemaLedgerInfo.size = metadata.getLength();
                        if (includeLedgerMetadata) {
                            info.metadata = metadata.toSafeString();
                        }
                        stats.schemaLedgers.add(schemaLedgerInfo);
                        completableFuture.complete(null);
                    }).exceptionally(e -> {
                        completableFuture.completeExceptionally(e);
                        return null;
                    });
                } else {
                    completableFuture.complete(null);
                }
            });
            FutureUtil.waitForAll(getLedgerMetadataFutures).thenRun(() -> {
                schemaStoreLedgersFuture.complete(null);
            }).exceptionally(e -> {
                schemaStoreLedgersFuture.completeExceptionally(e);
                return null;
            });
        }).exceptionally(e -> {
            schemaStoreLedgersFuture.completeExceptionally(e);
            return null;
        });
    } else {
        schemaStoreLedgersFuture.complete(null);
    }
    schemaStoreLedgersFuture.thenRun(() -> {
        if (futures != null) {
            FutureUtil.waitForAll(futures).handle((res, ex) -> {
                statFuture.complete(stats);
                return null;
            });
        } else {
            statFuture.complete(stats);
        }
    }).exceptionally(e -> {
        statFuture.completeExceptionally(e);
        return null;
    });
    return statFuture;
}
Also used : SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) Topic(org.apache.pulsar.broker.service.Topic) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) StringUtils(org.apache.commons.lang3.StringUtils) SubscribeRate(org.apache.pulsar.common.policies.data.SubscribeRate) MetadataNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) TxnID(org.apache.pulsar.client.api.transaction.TxnID) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) AlreadyRunningException(org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException) Map(java.util.Map) StatsOutputStream(org.apache.pulsar.utils.StatsOutputStream) EnumSet(java.util.EnumSet) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) UpdatePropertiesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.UpdatePropertiesCallback) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) ManagedLedgerAlreadyClosedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException) CancellationException(java.util.concurrent.CancellationException) ManagedCursorContainer(org.apache.bookkeeper.mledger.impl.ManagedCursorContainer) Set(java.util.Set) BrokerService(org.apache.pulsar.broker.service.BrokerService) NamespaceStats(org.apache.pulsar.broker.stats.NamespaceStats) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) OffloadProcessStatus(org.apache.pulsar.client.admin.OffloadProcessStatus) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) TransactionPendingAckStats(org.apache.pulsar.common.policies.data.TransactionPendingAckStats) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) TopicTerminatedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) ManagedLedgerTerminatedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) BookieId(org.apache.bookkeeper.net.BookieId) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) TransactionInPendingAckStats(org.apache.pulsar.common.policies.data.TransactionInPendingAckStats) Subscription(org.apache.pulsar.broker.service.Subscription) LedgerInfo(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo) NamespaceService(org.apache.pulsar.broker.namespace.NamespaceService) Consumer(org.apache.pulsar.broker.service.Consumer) TerminateCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback) ArrayList(java.util.ArrayList) TopicClosedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException) Commands(org.apache.pulsar.common.protocol.Commands) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Lists(com.google.common.collect.Lists) CursorStats(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats) DateFormatter(org.apache.pulsar.common.util.DateFormatter) TopicBacklogQuotaExceededException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBacklogQuotaExceededException) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) CompactorMXBean(org.apache.pulsar.compaction.CompactorMXBean) Dispatcher(org.apache.pulsar.broker.service.Dispatcher) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) BrokerServiceException(org.apache.pulsar.broker.service.BrokerServiceException) StreamingStats(org.apache.pulsar.broker.service.StreamingStats) AbstractReplicator(org.apache.pulsar.broker.service.AbstractReplicator) TopicFencedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Compactor(org.apache.pulsar.compaction.Compactor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) Codec(org.apache.pulsar.common.util.Codec) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) SystemTopicNames.isEventSystemTopic(org.apache.pulsar.common.naming.SystemTopicNames.isEventSystemTopic) ScheduledFuture(java.util.concurrent.ScheduledFuture) COMPACTION_SUBSCRIPTION(org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION) BookkeeperSchemaStorage(org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage) BiFunction(java.util.function.BiFunction) UnsupportedVersionException(org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) LoggerFactory(org.slf4j.LoggerFactory) SystemTopicNames(org.apache.pulsar.common.naming.SystemTopicNames) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) Type(org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type) BacklogQuotaType(org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) TxnAction(org.apache.pulsar.common.api.proto.TxnAction) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) TransactionInBufferStats(org.apache.pulsar.common.policies.data.TransactionInBufferStats) ReplicationMetrics(org.apache.pulsar.broker.stats.ReplicationMetrics) SubscribeRateLimiter.isSubscribeRateEnabled(org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter.isSubscribeRateEnabled) CompletionException(java.util.concurrent.CompletionException) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) CompactedTopicContext(org.apache.pulsar.compaction.CompactedTopicContext) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) InactiveTopicDeleteMode(org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) MLPendingAckStore(org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore) Optional(java.util.Optional) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) LedgerMetadata(org.apache.bookkeeper.client.api.LedgerMetadata) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) LongAdder(java.util.concurrent.atomic.LongAdder) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) CommandSubscribe(org.apache.pulsar.common.api.proto.CommandSubscribe) SubscriptionOption(org.apache.pulsar.broker.service.SubscriptionOption) ByteBuf(io.netty.buffer.ByteBuf) CompactedTopicImpl(org.apache.pulsar.compaction.CompactedTopicImpl) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) TransactionBufferDisable(org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable) LongRunningProcessStatus(org.apache.pulsar.client.admin.LongRunningProcessStatus) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) AbstractTopic(org.apache.pulsar.broker.service.AbstractTopic) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) MapUtils(org.apache.commons.collections4.MapUtils) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) TransportCnx(org.apache.pulsar.broker.service.TransportCnx) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) Logger(org.slf4j.Logger) CompactedTopic(org.apache.pulsar.compaction.CompactedTopic) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) OffloadCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OffloadCallback) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) Producer(org.apache.pulsar.broker.service.Producer) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) Maps(com.google.common.collect.Maps) TransactionBufferStats(org.apache.pulsar.common.policies.data.TransactionBufferStats) TransactionBuffer(org.apache.pulsar.broker.transaction.buffer.TransactionBuffer) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) MessageId(org.apache.pulsar.client.api.MessageId) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) Clock(java.time.Clock) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) Replicator(org.apache.pulsar.broker.service.Replicator) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) EnumSet(java.util.EnumSet) Set(java.util.Set) BookkeeperSchemaStorage(org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage) ArrayList(java.util.ArrayList) CompletableFuture(java.util.concurrent.CompletableFuture) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) ArrayList(java.util.ArrayList) List(java.util.List) CompactedTopicContext(org.apache.pulsar.compaction.CompactedTopicContext) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) LedgerInfo(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) CursorStats(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats)

Example 3 with BookkeeperSchemaStorage

use of org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage in project pulsar by yahoo.

the class PersistentTopic method getInternalStats.

@Override
public CompletableFuture<PersistentTopicInternalStats> getInternalStats(boolean includeLedgerMetadata) {
    CompletableFuture<PersistentTopicInternalStats> statFuture = new CompletableFuture<>();
    PersistentTopicInternalStats stats = new PersistentTopicInternalStats();
    ManagedLedgerImpl ml = (ManagedLedgerImpl) ledger;
    stats.entriesAddedCounter = ml.getEntriesAddedCounter();
    stats.numberOfEntries = ml.getNumberOfEntries();
    stats.totalSize = ml.getTotalSize();
    stats.currentLedgerEntries = ml.getCurrentLedgerEntries();
    stats.currentLedgerSize = ml.getCurrentLedgerSize();
    stats.lastLedgerCreatedTimestamp = DateFormatter.format(ml.getLastLedgerCreatedTimestamp());
    if (ml.getLastLedgerCreationFailureTimestamp() != 0) {
        stats.lastLedgerCreationFailureTimestamp = DateFormatter.format(ml.getLastLedgerCreationFailureTimestamp());
    }
    stats.waitingCursorsCount = ml.getWaitingCursorsCount();
    stats.pendingAddEntriesCount = ml.getPendingAddEntriesCount();
    stats.lastConfirmedEntry = ml.getLastConfirmedEntry().toString();
    stats.state = ml.getState().toString();
    stats.ledgers = Lists.newArrayList();
    List<CompletableFuture<String>> futures = Lists.newArrayList();
    CompletableFuture<Set<String>> availableBookiesFuture = brokerService.pulsar().getPulsarResources().getBookieResources().listAvailableBookiesAsync();
    futures.add(availableBookiesFuture.handle((strings, throwable) -> null));
    availableBookiesFuture.whenComplete((bookies, e) -> {
        if (e != null) {
            log.error("[{}] Failed to fetch available bookies.", topic, e);
            statFuture.completeExceptionally(e);
        } else {
            ml.getLedgersInfo().forEach((id, li) -> {
                LedgerInfo info = new LedgerInfo();
                info.ledgerId = li.getLedgerId();
                info.entries = li.getEntries();
                info.size = li.getSize();
                info.offloaded = li.hasOffloadContext() && li.getOffloadContext().getComplete();
                stats.ledgers.add(info);
                if (includeLedgerMetadata) {
                    futures.add(ml.getLedgerMetadata(li.getLedgerId()).handle((lMetadata, ex) -> {
                        if (ex == null) {
                            info.metadata = lMetadata;
                        }
                        return null;
                    }));
                    futures.add(ml.getEnsemblesAsync(li.getLedgerId()).handle((ensembles, ex) -> {
                        if (ex == null) {
                            info.underReplicated = !bookies.containsAll(ensembles.stream().map(BookieId::toString).collect(Collectors.toList()));
                        }
                        return null;
                    }));
                }
            });
        }
    });
    // Add ledger info for compacted topic ledger if exist.
    LedgerInfo info = new LedgerInfo();
    info.ledgerId = -1;
    info.entries = -1;
    info.size = -1;
    Optional<CompactedTopicContext> compactedTopicContext = getCompactedTopicContext();
    if (compactedTopicContext.isPresent()) {
        CompactedTopicContext ledgerContext = compactedTopicContext.get();
        info.ledgerId = ledgerContext.getLedger().getId();
        info.entries = ledgerContext.getLedger().getLastAddConfirmed() + 1;
        info.size = ledgerContext.getLedger().getLength();
    }
    stats.compactedLedger = info;
    stats.cursors = Maps.newTreeMap();
    ml.getCursors().forEach(c -> {
        ManagedCursorImpl cursor = (ManagedCursorImpl) c;
        CursorStats cs = new CursorStats();
        cs.markDeletePosition = cursor.getMarkDeletedPosition().toString();
        cs.readPosition = cursor.getReadPosition().toString();
        cs.waitingReadOp = cursor.hasPendingReadRequest();
        cs.pendingReadOps = cursor.getPendingReadOpsCount();
        cs.messagesConsumedCounter = cursor.getMessagesConsumedCounter();
        cs.cursorLedger = cursor.getCursorLedger();
        cs.cursorLedgerLastEntry = cursor.getCursorLedgerLastEntry();
        cs.individuallyDeletedMessages = cursor.getIndividuallyDeletedMessages();
        cs.lastLedgerSwitchTimestamp = DateFormatter.format(cursor.getLastLedgerSwitchTimestamp());
        cs.state = cursor.getState();
        cs.numberOfEntriesSinceFirstNotAckedMessage = cursor.getNumberOfEntriesSinceFirstNotAckedMessage();
        cs.totalNonContiguousDeletedMessagesRange = cursor.getTotalNonContiguousDeletedMessagesRange();
        cs.properties = cursor.getProperties();
        // subscription metrics
        PersistentSubscription sub = subscriptions.get(Codec.decode(c.getName()));
        if (sub != null) {
            if (sub.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) {
                PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) sub.getDispatcher();
                cs.subscriptionHavePendingRead = dispatcher.havePendingRead;
                cs.subscriptionHavePendingReplayRead = dispatcher.havePendingReplayRead;
            } else if (sub.getDispatcher() instanceof PersistentDispatcherSingleActiveConsumer) {
                PersistentDispatcherSingleActiveConsumer dispatcher = (PersistentDispatcherSingleActiveConsumer) sub.getDispatcher();
                cs.subscriptionHavePendingRead = dispatcher.havePendingRead;
            }
        }
        stats.cursors.put(cursor.getName(), cs);
    });
    // Schema store ledgers
    String schemaId;
    try {
        schemaId = TopicName.get(topic).getSchemaName();
    } catch (Throwable t) {
        statFuture.completeExceptionally(t);
        return statFuture;
    }
    CompletableFuture<Void> schemaStoreLedgersFuture = new CompletableFuture<>();
    stats.schemaLedgers = Collections.synchronizedList(new ArrayList<>());
    if (brokerService.getPulsar().getSchemaStorage() != null && brokerService.getPulsar().getSchemaStorage() instanceof BookkeeperSchemaStorage) {
        ((BookkeeperSchemaStorage) brokerService.getPulsar().getSchemaStorage()).getStoreLedgerIdsBySchemaId(schemaId).thenAccept(ledgers -> {
            List<CompletableFuture<Void>> getLedgerMetadataFutures = new ArrayList<>();
            ledgers.forEach(ledgerId -> {
                CompletableFuture<Void> completableFuture = new CompletableFuture<>();
                getLedgerMetadataFutures.add(completableFuture);
                CompletableFuture<LedgerMetadata> metadataFuture = null;
                try {
                    metadataFuture = brokerService.getPulsar().getBookKeeperClient().getLedgerMetadata(ledgerId);
                } catch (NullPointerException e) {
                    // related to bookkeeper issue https://github.com/apache/bookkeeper/issues/2741
                    if (log.isDebugEnabled()) {
                        log.debug("{{}} Failed to get ledger metadata for the schema ledger {}", topic, ledgerId, e);
                    }
                }
                if (metadataFuture != null) {
                    metadataFuture.thenAccept(metadata -> {
                        LedgerInfo schemaLedgerInfo = new LedgerInfo();
                        schemaLedgerInfo.ledgerId = metadata.getLedgerId();
                        schemaLedgerInfo.entries = metadata.getLastEntryId() + 1;
                        schemaLedgerInfo.size = metadata.getLength();
                        if (includeLedgerMetadata) {
                            info.metadata = metadata.toSafeString();
                        }
                        stats.schemaLedgers.add(schemaLedgerInfo);
                        completableFuture.complete(null);
                    }).exceptionally(e -> {
                        completableFuture.completeExceptionally(e);
                        return null;
                    });
                } else {
                    completableFuture.complete(null);
                }
            });
            FutureUtil.waitForAll(getLedgerMetadataFutures).thenRun(() -> {
                schemaStoreLedgersFuture.complete(null);
            }).exceptionally(e -> {
                schemaStoreLedgersFuture.completeExceptionally(e);
                return null;
            });
        }).exceptionally(e -> {
            schemaStoreLedgersFuture.completeExceptionally(e);
            return null;
        });
    } else {
        schemaStoreLedgersFuture.complete(null);
    }
    schemaStoreLedgersFuture.thenRun(() -> {
        if (futures != null) {
            FutureUtil.waitForAll(futures).handle((res, ex) -> {
                statFuture.complete(stats);
                return null;
            });
        } else {
            statFuture.complete(stats);
        }
    }).exceptionally(e -> {
        statFuture.completeExceptionally(e);
        return null;
    });
    return statFuture;
}
Also used : SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) Topic(org.apache.pulsar.broker.service.Topic) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) StringUtils(org.apache.commons.lang3.StringUtils) SubscribeRate(org.apache.pulsar.common.policies.data.SubscribeRate) MetadataNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) TxnID(org.apache.pulsar.client.api.transaction.TxnID) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) AlreadyRunningException(org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException) Map(java.util.Map) StatsOutputStream(org.apache.pulsar.utils.StatsOutputStream) EnumSet(java.util.EnumSet) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) UpdatePropertiesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.UpdatePropertiesCallback) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) ManagedLedgerAlreadyClosedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException) CancellationException(java.util.concurrent.CancellationException) ManagedCursorContainer(org.apache.bookkeeper.mledger.impl.ManagedCursorContainer) Set(java.util.Set) BrokerService(org.apache.pulsar.broker.service.BrokerService) NamespaceStats(org.apache.pulsar.broker.stats.NamespaceStats) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) OffloadProcessStatus(org.apache.pulsar.client.admin.OffloadProcessStatus) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) TransactionPendingAckStats(org.apache.pulsar.common.policies.data.TransactionPendingAckStats) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) TopicTerminatedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) ManagedLedgerTerminatedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) BookieId(org.apache.bookkeeper.net.BookieId) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) TransactionInPendingAckStats(org.apache.pulsar.common.policies.data.TransactionInPendingAckStats) Subscription(org.apache.pulsar.broker.service.Subscription) LedgerInfo(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo) NamespaceService(org.apache.pulsar.broker.namespace.NamespaceService) Consumer(org.apache.pulsar.broker.service.Consumer) TerminateCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback) ArrayList(java.util.ArrayList) TopicClosedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException) Commands(org.apache.pulsar.common.protocol.Commands) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Lists(com.google.common.collect.Lists) CursorStats(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats) DateFormatter(org.apache.pulsar.common.util.DateFormatter) TopicBacklogQuotaExceededException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBacklogQuotaExceededException) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) CompactorMXBean(org.apache.pulsar.compaction.CompactorMXBean) Dispatcher(org.apache.pulsar.broker.service.Dispatcher) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) BrokerServiceException(org.apache.pulsar.broker.service.BrokerServiceException) StreamingStats(org.apache.pulsar.broker.service.StreamingStats) AbstractReplicator(org.apache.pulsar.broker.service.AbstractReplicator) TopicFencedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Compactor(org.apache.pulsar.compaction.Compactor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) Codec(org.apache.pulsar.common.util.Codec) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) SystemTopicNames.isEventSystemTopic(org.apache.pulsar.common.naming.SystemTopicNames.isEventSystemTopic) ScheduledFuture(java.util.concurrent.ScheduledFuture) COMPACTION_SUBSCRIPTION(org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION) BookkeeperSchemaStorage(org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage) BiFunction(java.util.function.BiFunction) UnsupportedVersionException(org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) LoggerFactory(org.slf4j.LoggerFactory) SystemTopicNames(org.apache.pulsar.common.naming.SystemTopicNames) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) Type(org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type) BacklogQuotaType(org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) TxnAction(org.apache.pulsar.common.api.proto.TxnAction) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) TransactionInBufferStats(org.apache.pulsar.common.policies.data.TransactionInBufferStats) ReplicationMetrics(org.apache.pulsar.broker.stats.ReplicationMetrics) SubscribeRateLimiter.isSubscribeRateEnabled(org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter.isSubscribeRateEnabled) CompletionException(java.util.concurrent.CompletionException) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) CompactedTopicContext(org.apache.pulsar.compaction.CompactedTopicContext) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) InactiveTopicDeleteMode(org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) MLPendingAckStore(org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore) Optional(java.util.Optional) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) LedgerMetadata(org.apache.bookkeeper.client.api.LedgerMetadata) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) LongAdder(java.util.concurrent.atomic.LongAdder) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) CommandSubscribe(org.apache.pulsar.common.api.proto.CommandSubscribe) SubscriptionOption(org.apache.pulsar.broker.service.SubscriptionOption) ByteBuf(io.netty.buffer.ByteBuf) CompactedTopicImpl(org.apache.pulsar.compaction.CompactedTopicImpl) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) TransactionBufferDisable(org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable) LongRunningProcessStatus(org.apache.pulsar.client.admin.LongRunningProcessStatus) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) AbstractTopic(org.apache.pulsar.broker.service.AbstractTopic) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) MapUtils(org.apache.commons.collections4.MapUtils) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) TransportCnx(org.apache.pulsar.broker.service.TransportCnx) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) Logger(org.slf4j.Logger) CompactedTopic(org.apache.pulsar.compaction.CompactedTopic) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) OffloadCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OffloadCallback) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) Producer(org.apache.pulsar.broker.service.Producer) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) Maps(com.google.common.collect.Maps) TransactionBufferStats(org.apache.pulsar.common.policies.data.TransactionBufferStats) TransactionBuffer(org.apache.pulsar.broker.transaction.buffer.TransactionBuffer) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) MessageId(org.apache.pulsar.client.api.MessageId) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) Clock(java.time.Clock) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) Replicator(org.apache.pulsar.broker.service.Replicator) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) EnumSet(java.util.EnumSet) Set(java.util.Set) BookkeeperSchemaStorage(org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage) ArrayList(java.util.ArrayList) CompletableFuture(java.util.concurrent.CompletableFuture) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) ArrayList(java.util.ArrayList) List(java.util.List) CompactedTopicContext(org.apache.pulsar.compaction.CompactedTopicContext) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) LedgerInfo(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) CursorStats(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats)

Aggregations

ObjectObjectHashMap (com.carrotsearch.hppc.ObjectObjectHashMap)3 VisibleForTesting (com.google.common.annotations.VisibleForTesting)3 Preconditions.checkNotNull (com.google.common.base.Preconditions.checkNotNull)3 Lists (com.google.common.collect.Lists)3 Maps (com.google.common.collect.Maps)3 Sets (com.google.common.collect.Sets)3 ByteBuf (io.netty.buffer.ByteBuf)3 FastThreadLocal (io.netty.util.concurrent.FastThreadLocal)3 Clock (java.time.Clock)3 ArrayList (java.util.ArrayList)3 Collections (java.util.Collections)3 EnumSet (java.util.EnumSet)3 List (java.util.List)3 Map (java.util.Map)3 Optional (java.util.Optional)3 Set (java.util.Set)3 CancellationException (java.util.concurrent.CancellationException)3 CompletableFuture (java.util.concurrent.CompletableFuture)3 CompletionException (java.util.concurrent.CompletionException)3 ExecutionException (java.util.concurrent.ExecutionException)3