Examples with ConsumerStatsImpl org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl used on opensource projects

Example 6 with ConsumerStatsImpl

use of org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl in project pulsar by apache.

the class ConsumerStatsTest method testUpdateStatsForActiveConsumerAndSubscription.

@Test
public void testUpdateStatsForActiveConsumerAndSubscription() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/testUpdateStatsForActiveConsumerAndSubscription";
    pulsarClient.newConsumer().topic(topicName).subscriptionType(SubscriptionType.Shared).subscriptionName("my-subscription").subscribe();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName).get();
    Assert.assertNotNull(topicRef);
    Assert.assertEquals(topicRef.getSubscriptions().size(), 1);
    List<org.apache.pulsar.broker.service.Consumer> consumers = topicRef.getSubscriptions().get("my-subscription").getConsumers();
    Assert.assertEquals(consumers.size(), 1);
    ConsumerStatsImpl consumerStats = new ConsumerStatsImpl();
    consumerStats.msgOutCounter = 10;
    consumerStats.bytesOutCounter = 1280;
    consumers.get(0).updateStats(consumerStats);
    ConsumerStats updatedStats = consumers.get(0).getStats();
    Assert.assertEquals(updatedStats.getMsgOutCounter(), 10);
    Assert.assertEquals(updatedStats.getBytesOutCounter(), 1280);
}

Example 7 with ConsumerStatsImpl

use of org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl in project pulsar by apache.

the class NamespaceStatsAggregator method getTopicStats.

private static void getTopicStats(Topic topic, TopicStats stats, boolean includeConsumerMetrics, boolean includeProducerMetrics, boolean getPreciseBacklog, boolean subscriptionBacklogSize, Optional<CompactorMXBean> compactorMXBean) {
    stats.reset();
    if (topic instanceof PersistentTopic) {
        // Managed Ledger stats
        ManagedLedger ml = ((PersistentTopic) topic).getManagedLedger();
        ManagedLedgerMBeanImpl mlStats = (ManagedLedgerMBeanImpl) ml.getStats();
        stats.managedLedgerStats.storageSize = mlStats.getStoredMessagesSize();
        stats.managedLedgerStats.storageLogicalSize = mlStats.getStoredMessagesLogicalSize();
        stats.managedLedgerStats.backlogSize = ml.getEstimatedBacklogSize();
        stats.managedLedgerStats.offloadedStorageUsed = ml.getOffloadedSize();
        stats.backlogQuotaLimit = topic.getBacklogQuota(BacklogQuota.BacklogQuotaType.destination_storage).getLimitSize();
        stats.backlogQuotaLimitTime = topic.getBacklogQuota(BacklogQuota.BacklogQuotaType.message_age).getLimitTime();
        stats.managedLedgerStats.storageWriteLatencyBuckets.addAll(mlStats.getInternalAddEntryLatencyBuckets());
        stats.managedLedgerStats.storageWriteLatencyBuckets.refresh();
        stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.addAll(mlStats.getInternalLedgerAddEntryLatencyBuckets());
        stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.refresh();
        stats.managedLedgerStats.entrySizeBuckets.addAll(mlStats.getInternalEntrySizeBuckets());
        stats.managedLedgerStats.entrySizeBuckets.refresh();
        stats.managedLedgerStats.storageWriteRate = mlStats.getAddEntryMessagesRate();
        stats.managedLedgerStats.storageReadRate = mlStats.getReadEntriesRate();
    }
    TopicStatsImpl tStatus = topic.getStats(getPreciseBacklog, subscriptionBacklogSize, false);
    stats.msgInCounter = tStatus.msgInCounter;
    stats.bytesInCounter = tStatus.bytesInCounter;
    stats.msgOutCounter = tStatus.msgOutCounter;
    stats.bytesOutCounter = tStatus.bytesOutCounter;
    stats.averageMsgSize = tStatus.averageMsgSize;
    stats.publishRateLimitedTimes = tStatus.publishRateLimitedTimes;
    stats.producersCount = 0;
    topic.getProducers().values().forEach(producer -> {
        if (producer.isRemote()) {
            AggregatedReplicationStats replStats = stats.replicationStats.computeIfAbsent(producer.getRemoteCluster(), k -> new AggregatedReplicationStats());
            replStats.msgRateIn += producer.getStats().msgRateIn;
            replStats.msgThroughputIn += producer.getStats().msgThroughputIn;
        } else {
            // Local producer
            stats.producersCount++;
            stats.rateIn += producer.getStats().msgRateIn;
            stats.throughputIn += producer.getStats().msgThroughputIn;
            if (includeProducerMetrics) {
                AggregatedProducerStats producerStats = stats.producerStats.computeIfAbsent(producer.getProducerName(), k -> new AggregatedProducerStats());
                producerStats.producerId = producer.getStats().producerId;
                producerStats.msgRateIn = producer.getStats().msgRateIn;
                producerStats.msgThroughputIn = producer.getStats().msgThroughputIn;
                producerStats.averageMsgSize = producer.getStats().averageMsgSize;
            }
        }
    });
    if (topic instanceof PersistentTopic) {
        tStatus.subscriptions.forEach((subName, subscriptionStats) -> {
            AggregatedSubscriptionStats subsStats = stats.subscriptionStats.computeIfAbsent(subName, k -> new AggregatedSubscriptionStats());
            aggregateTopicStats(stats, subscriptionStats, subsStats);
        });
    } else {
        ((NonPersistentTopicStatsImpl) tStatus).getNonPersistentSubscriptions().forEach((subName, nonPersistentSubscriptionStats) -> {
            NonPersistentSubscriptionStatsImpl subscriptionStats = (NonPersistentSubscriptionStatsImpl) nonPersistentSubscriptionStats;
            AggregatedSubscriptionStats subsStats = stats.subscriptionStats.computeIfAbsent(subName, k -> new AggregatedSubscriptionStats());
            aggregateTopicStats(stats, subscriptionStats, subsStats);
            subsStats.msgDropRate += subscriptionStats.getMsgDropRate();
        });
    }
    // Consumer stats can be a lot if a subscription has many consumers
    if (includeConsumerMetrics) {
        topic.getSubscriptions().forEach((name, subscription) -> {
            AggregatedSubscriptionStats subsStats = stats.subscriptionStats.computeIfAbsent(name, k -> new AggregatedSubscriptionStats());
            subscription.getConsumers().forEach(consumer -> {
                ConsumerStatsImpl conStats = consumer.getStats();
                AggregatedConsumerStats consumerStats = subsStats.consumerStat.computeIfAbsent(consumer, k -> new AggregatedConsumerStats());
                consumerStats.unackedMessages = conStats.unackedMessages;
                consumerStats.msgRateRedeliver = conStats.msgRateRedeliver;
                consumerStats.msgRateOut = conStats.msgRateOut;
                consumerStats.msgThroughputOut = conStats.msgThroughputOut;
                consumerStats.bytesOutCounter = conStats.bytesOutCounter;
                consumerStats.msgOutCounter = conStats.msgOutCounter;
                consumerStats.availablePermits = conStats.availablePermits;
                consumerStats.blockedSubscriptionOnUnackedMsgs = conStats.blockedConsumerOnUnackedMsgs;
            });
        });
    }
    topic.getReplicators().forEach((cluster, replicator) -> {
        AggregatedReplicationStats aggReplStats = stats.replicationStats.computeIfAbsent(cluster, k -> new AggregatedReplicationStats());
        ReplicatorStatsImpl replStats = replicator.getStats();
        aggReplStats.msgRateOut += replStats.msgRateOut;
        aggReplStats.msgThroughputOut += replStats.msgThroughputOut;
        aggReplStats.replicationBacklog += replStats.replicationBacklog;
        aggReplStats.msgRateIn += replStats.msgRateIn;
        aggReplStats.msgThroughputIn += replStats.msgThroughputIn;
        aggReplStats.msgRateExpired += replStats.msgRateExpired;
        aggReplStats.connectedCount += replStats.connected ? 1 : 0;
        aggReplStats.replicationDelayInSeconds += replStats.replicationDelayInSeconds;
    });
    compactorMXBean.flatMap(mxBean -> mxBean.getCompactionRecordForTopic(topic.getName())).map(compactionRecord -> {
        stats.compactionRemovedEventCount = compactionRecord.getCompactionRemovedEventCount();
        stats.compactionSucceedCount = compactionRecord.getCompactionSucceedCount();
        stats.compactionFailedCount = compactionRecord.getCompactionFailedCount();
        stats.compactionDurationTimeInMills = compactionRecord.getCompactionDurationTimeInMills();
        stats.compactionReadThroughput = compactionRecord.getCompactionReadThroughput();
        stats.compactionWriteThroughput = compactionRecord.getCompactionWriteThroughput();
        stats.compactionLatencyBuckets.addAll(compactionRecord.getCompactionLatencyStats());
        stats.compactionLatencyBuckets.refresh();
        PersistentTopic persistentTopic = (PersistentTopic) topic;
        Optional<CompactedTopicContext> compactedTopicContext = persistentTopic.getCompactedTopicContext();
        if (compactedTopicContext.isPresent()) {
            LedgerHandle ledger = compactedTopicContext.get().getLedger();
            long entries = ledger.getLastAddConfirmed() + 1;
            long size = ledger.getLength();
            stats.compactionCompactedEntriesCount = entries;
            stats.compactionCompactedEntriesSize = size;
        }
        return compactionRecord;
    });
}

Example 8 with ConsumerStatsImpl

use of org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl in project pulsar by apache.

the class PersistentTopic method updateRates.

@Override
public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats, StatsOutputStream topicStatsStream, ClusterReplicationMetrics replStats, String namespace, boolean hydratePublishers) {
    this.publishRateLimitedTimes = 0;
    TopicStatsHelper topicStatsHelper = threadLocalTopicStats.get();
    topicStatsHelper.reset();
    replicators.forEach((region, replicator) -> replicator.updateRates());
    nsStats.producerCount += producers.size();
    bundleStats.producerCount += producers.size();
    topicStatsStream.startObject(topic);
    // start publisher stats
    topicStatsStream.startList("publishers");
    producers.values().forEach(producer -> {
        producer.updateRates();
        PublisherStatsImpl publisherStats = producer.getStats();
        topicStatsHelper.aggMsgRateIn += publisherStats.msgRateIn;
        topicStatsHelper.aggMsgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            topicStatsHelper.remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        }
        // Populate consumer specific stats here
        if (hydratePublishers) {
            StreamingStats.writePublisherStats(topicStatsStream, publisherStats);
        }
    });
    topicStatsStream.endList();
    // if publish-rate increases (eg: 0 to 1K) then pick max publish-rate and if publish-rate decreases then keep
    // average rate.
    lastUpdatedAvgPublishRateInMsg = topicStatsHelper.aggMsgRateIn > lastUpdatedAvgPublishRateInMsg ? topicStatsHelper.aggMsgRateIn : (topicStatsHelper.aggMsgRateIn + lastUpdatedAvgPublishRateInMsg) / 2;
    lastUpdatedAvgPublishRateInByte = topicStatsHelper.aggMsgThroughputIn > lastUpdatedAvgPublishRateInByte ? topicStatsHelper.aggMsgThroughputIn : (topicStatsHelper.aggMsgThroughputIn + lastUpdatedAvgPublishRateInByte) / 2;
    // Start replicator stats
    topicStatsStream.startObject("replication");
    nsStats.replicatorCount += topicStatsHelper.remotePublishersStats.size();
    replicators.forEach((cluster, replicator) -> {
        // Update replicator cursor state
        try {
            ((PersistentReplicator) replicator).updateCursorState();
        } catch (Exception e) {
            log.warn("[{}] Failed to update cursor state ", topic, e);
        }
        // Update replicator stats
        ReplicatorStatsImpl rStat = replicator.getStats();
        // Add incoming msg rates
        PublisherStatsImpl pubStats = topicStatsHelper.remotePublishersStats.get(replicator.getRemoteCluster());
        rStat.msgRateIn = pubStats != null ? pubStats.msgRateIn : 0;
        rStat.msgThroughputIn = pubStats != null ? pubStats.msgThroughputIn : 0;
        rStat.inboundConnection = pubStats != null ? pubStats.getAddress() : null;
        rStat.inboundConnectedSince = pubStats != null ? pubStats.getConnectedSince() : null;
        topicStatsHelper.aggMsgRateOut += rStat.msgRateOut;
        topicStatsHelper.aggMsgThroughputOut += rStat.msgThroughputOut;
        // Populate replicator specific stats here
        topicStatsStream.startObject(cluster);
        topicStatsStream.writePair("connected", rStat.connected);
        topicStatsStream.writePair("msgRateExpired", rStat.msgRateExpired);
        topicStatsStream.writePair("msgRateIn", rStat.msgRateIn);
        topicStatsStream.writePair("msgRateOut", rStat.msgRateOut);
        topicStatsStream.writePair("msgThroughputIn", rStat.msgThroughputIn);
        topicStatsStream.writePair("msgThroughputOut", rStat.msgThroughputOut);
        topicStatsStream.writePair("replicationBacklog", rStat.replicationBacklog);
        topicStatsStream.writePair("replicationDelayInSeconds", rStat.replicationDelayInSeconds);
        topicStatsStream.writePair("inboundConnection", rStat.inboundConnection);
        topicStatsStream.writePair("inboundConnectedSince", rStat.inboundConnectedSince);
        topicStatsStream.writePair("outboundConnection", rStat.outboundConnection);
        topicStatsStream.writePair("outboundConnectedSince", rStat.outboundConnectedSince);
        topicStatsStream.endObject();
        nsStats.msgReplBacklog += rStat.replicationBacklog;
        if (replStats.isMetricsEnabled()) {
            String namespaceClusterKey = replStats.getKeyName(namespace, cluster);
            ReplicationMetrics replicationMetrics = replStats.get(namespaceClusterKey);
            boolean update = false;
            if (replicationMetrics == null) {
                replicationMetrics = ReplicationMetrics.get();
                update = true;
            }
            replicationMetrics.connected += rStat.connected ? 1 : 0;
            replicationMetrics.msgRateOut += rStat.msgRateOut;
            replicationMetrics.msgThroughputOut += rStat.msgThroughputOut;
            replicationMetrics.msgReplBacklog += rStat.replicationBacklog;
            if (update) {
                replStats.put(namespaceClusterKey, replicationMetrics);
            }
            // replication delay for a namespace is the max repl-delay among all the topics under this namespace
            if (rStat.replicationDelayInSeconds > replicationMetrics.maxMsgReplDelayInSeconds) {
                replicationMetrics.maxMsgReplDelayInSeconds = rStat.replicationDelayInSeconds;
            }
        }
    });
    // Close replication
    topicStatsStream.endObject();
    // Start subscription stats
    topicStatsStream.startObject("subscriptions");
    nsStats.subsCount += subscriptions.size();
    subscriptions.forEach((subscriptionName, subscription) -> {
        double subMsgRateOut = 0;
        double subMsgThroughputOut = 0;
        double subMsgRateRedeliver = 0;
        // Start subscription name & consumers
        try {
            topicStatsStream.startObject(subscriptionName);
            topicStatsStream.startList("consumers");
            for (Consumer consumer : subscription.getConsumers()) {
                ++nsStats.consumerCount;
                ++bundleStats.consumerCount;
                consumer.updateRates();
                ConsumerStatsImpl consumerStats = consumer.getStats();
                subMsgRateOut += consumerStats.msgRateOut;
                subMsgThroughputOut += consumerStats.msgThroughputOut;
                subMsgRateRedeliver += consumerStats.msgRateRedeliver;
                StreamingStats.writeConsumerStats(topicStatsStream, subscription.getType(), consumerStats);
            }
            // Close Consumer stats
            topicStatsStream.endList();
            // Populate subscription specific stats here
            topicStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog(true));
            topicStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate());
            topicStatsStream.writePair("msgRateOut", subMsgRateOut);
            topicStatsStream.writePair("msgThroughputOut", subMsgThroughputOut);
            topicStatsStream.writePair("msgRateRedeliver", subMsgRateRedeliver);
            topicStatsStream.writePair("numberOfEntriesSinceFirstNotAckedMessage", subscription.getNumberOfEntriesSinceFirstNotAckedMessage());
            topicStatsStream.writePair("totalNonContiguousDeletedMessagesRange", subscription.getTotalNonContiguousDeletedMessagesRange());
            topicStatsStream.writePair("type", subscription.getTypeString());
            if (Subscription.isIndividualAckMode(subscription.getType())) {
                if (subscription.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) {
                    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subscription.getDispatcher();
                    topicStatsStream.writePair("blockedSubscriptionOnUnackedMsgs", dispatcher.isBlockedDispatcherOnUnackedMsgs());
                    topicStatsStream.writePair("unackedMessages", dispatcher.getTotalUnackedMessages());
                }
            }
            // Close consumers
            topicStatsStream.endObject();
            topicStatsHelper.aggMsgRateOut += subMsgRateOut;
            topicStatsHelper.aggMsgThroughputOut += subMsgThroughputOut;
            nsStats.msgBacklog += subscription.getNumberOfEntriesInBacklog(false);
            // check stuck subscription
            if (brokerService.getPulsar().getConfig().isUnblockStuckSubscriptionEnabled()) {
                subscription.checkAndUnblockIfStuck();
            }
        } catch (Exception e) {
            log.error("Got exception when creating consumer stats for subscription {}: {}", subscriptionName, e.getMessage(), e);
        }
    });
    // Close subscription
    topicStatsStream.endObject();
    // Remaining dest stats.
    topicStatsHelper.averageMsgSize = topicStatsHelper.aggMsgRateIn == 0.0 ? 0.0 : (topicStatsHelper.aggMsgThroughputIn / topicStatsHelper.aggMsgRateIn);
    topicStatsStream.writePair("producerCount", producers.size());
    topicStatsStream.writePair("averageMsgSize", topicStatsHelper.averageMsgSize);
    topicStatsStream.writePair("msgRateIn", topicStatsHelper.aggMsgRateIn);
    topicStatsStream.writePair("msgRateOut", topicStatsHelper.aggMsgRateOut);
    topicStatsStream.writePair("msgInCount", getMsgInCounter());
    topicStatsStream.writePair("bytesInCount", getBytesInCounter());
    topicStatsStream.writePair("msgOutCount", getMsgOutCounter());
    topicStatsStream.writePair("bytesOutCount", getBytesOutCounter());
    topicStatsStream.writePair("msgThroughputIn", topicStatsHelper.aggMsgThroughputIn);
    topicStatsStream.writePair("msgThroughputOut", topicStatsHelper.aggMsgThroughputOut);
    topicStatsStream.writePair("storageSize", ledger.getTotalSize());
    topicStatsStream.writePair("backlogSize", ledger.getEstimatedBacklogSize());
    topicStatsStream.writePair("pendingAddEntriesCount", ((ManagedLedgerImpl) ledger).getPendingAddEntriesCount());
    nsStats.msgRateIn += topicStatsHelper.aggMsgRateIn;
    nsStats.msgRateOut += topicStatsHelper.aggMsgRateOut;
    nsStats.msgThroughputIn += topicStatsHelper.aggMsgThroughputIn;
    nsStats.msgThroughputOut += topicStatsHelper.aggMsgThroughputOut;
    nsStats.storageSize += ledger.getEstimatedBacklogSize();
    bundleStats.msgRateIn += topicStatsHelper.aggMsgRateIn;
    bundleStats.msgRateOut += topicStatsHelper.aggMsgRateOut;
    bundleStats.msgThroughputIn += topicStatsHelper.aggMsgThroughputIn;
    bundleStats.msgThroughputOut += topicStatsHelper.aggMsgThroughputOut;
    bundleStats.cacheSize += ((ManagedLedgerImpl) ledger).getCacheSize();
    // Close topic object
    topicStatsStream.endObject();
    // add publish-latency metrics
    this.addEntryLatencyStatsUsec.refresh();
    NamespaceStats.add(this.addEntryLatencyStatsUsec.getBuckets(), nsStats.addLatencyBucket);
    this.addEntryLatencyStatsUsec.reset();
}

Example 9 with ConsumerStatsImpl

use of org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl in project pulsar by apache.

the class PersistentSubscription method getStats.

public SubscriptionStatsImpl getStats(Boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) {
    SubscriptionStatsImpl subStats = new SubscriptionStatsImpl();
    subStats.lastExpireTimestamp = lastExpireTimestamp;
    subStats.lastConsumedFlowTimestamp = lastConsumedFlowTimestamp;
    subStats.lastMarkDeleteAdvancedTimestamp = lastMarkDeleteAdvancedTimestamp;
    subStats.bytesOutCounter = bytesOutFromRemovedConsumers.longValue();
    subStats.msgOutCounter = msgOutFromRemovedConsumer.longValue();
    Dispatcher dispatcher = this.dispatcher;
    if (dispatcher != null) {
        Map<Consumer, List<Range>> consumerKeyHashRanges = getType() == SubType.Key_Shared ? ((PersistentStickyKeyDispatcherMultipleConsumers) dispatcher).getConsumerKeyHashRanges() : null;
        dispatcher.getConsumers().forEach(consumer -> {
            ConsumerStatsImpl consumerStats = consumer.getStats();
            subStats.consumers.add(consumerStats);
            subStats.msgRateOut += consumerStats.msgRateOut;
            subStats.msgThroughputOut += consumerStats.msgThroughputOut;
            subStats.bytesOutCounter += consumerStats.bytesOutCounter;
            subStats.msgOutCounter += consumerStats.msgOutCounter;
            subStats.msgRateRedeliver += consumerStats.msgRateRedeliver;
            subStats.chunkedMessageRate += consumerStats.chunkedMessageRate;
            subStats.unackedMessages += consumerStats.unackedMessages;
            subStats.lastConsumedTimestamp = Math.max(subStats.lastConsumedTimestamp, consumerStats.lastConsumedTimestamp);
            subStats.lastAckedTimestamp = Math.max(subStats.lastAckedTimestamp, consumerStats.lastAckedTimestamp);
            if (consumerKeyHashRanges != null && consumerKeyHashRanges.containsKey(consumer)) {
                consumerStats.keyHashRanges = consumerKeyHashRanges.get(consumer).stream().map(Range::toString).collect(Collectors.toList());
            }
        });
    }
    SubType subType = getType();
    subStats.type = getTypeString();
    if (dispatcher instanceof PersistentDispatcherSingleActiveConsumer) {
        Consumer activeConsumer = ((PersistentDispatcherSingleActiveConsumer) dispatcher).getActiveConsumer();
        if (activeConsumer != null) {
            subStats.activeConsumerName = activeConsumer.consumerName();
        }
    }
    if (Subscription.isIndividualAckMode(subType)) {
        if (dispatcher instanceof PersistentDispatcherMultipleConsumers) {
            PersistentDispatcherMultipleConsumers d = (PersistentDispatcherMultipleConsumers) dispatcher;
            subStats.unackedMessages = d.getTotalUnackedMessages();
            subStats.blockedSubscriptionOnUnackedMsgs = d.isBlockedDispatcherOnUnackedMsgs();
            subStats.msgDelayed = d.getNumberOfDelayedMessages();
        }
    }
    subStats.msgBacklog = getNumberOfEntriesInBacklog(getPreciseBacklog);
    if (subscriptionBacklogSize) {
        subStats.backlogSize = ((ManagedLedgerImpl) topic.getManagedLedger()).getEstimatedBacklogSize((PositionImpl) cursor.getMarkDeletedPosition());
    }
    if (getEarliestTimeInBacklog && subStats.msgBacklog > 0) {
        ManagedLedgerImpl managedLedger = ((ManagedLedgerImpl) cursor.getManagedLedger());
        PositionImpl markDeletedPosition = (PositionImpl) cursor.getMarkDeletedPosition();
        long result = 0;
        try {
            result = managedLedger.getEarliestMessagePublishTimeOfPos(markDeletedPosition).get();
        } catch (InterruptedException | ExecutionException e) {
            result = -1;
        }
        subStats.earliestMsgPublishTimeInBacklog = result;
    }
    subStats.msgBacklogNoDelayed = subStats.msgBacklog - subStats.msgDelayed;
    subStats.msgRateExpired = expiryMonitor.getMessageExpiryRate();
    subStats.totalMsgExpired = expiryMonitor.getTotalMessageExpired();
    subStats.isReplicated = isReplicated();
    subStats.subscriptionProperties = subscriptionProperties;
    subStats.isDurable = cursor.isDurable();
    if (getType() == SubType.Key_Shared && dispatcher instanceof PersistentStickyKeyDispatcherMultipleConsumers) {
        PersistentStickyKeyDispatcherMultipleConsumers keySharedDispatcher = (PersistentStickyKeyDispatcherMultipleConsumers) dispatcher;
        subStats.allowOutOfOrderDelivery = keySharedDispatcher.isAllowOutOfOrderDelivery();
        subStats.keySharedMode = keySharedDispatcher.getKeySharedMode().toString();
        LinkedHashMap<Consumer, PositionImpl> recentlyJoinedConsumers = keySharedDispatcher.getRecentlyJoinedConsumers();
        if (recentlyJoinedConsumers != null && recentlyJoinedConsumers.size() > 0) {
            recentlyJoinedConsumers.forEach((k, v) -> {
                subStats.consumersAfterMarkDeletePosition.put(k.consumerName(), v.toString());
            });
        }
    }
    subStats.nonContiguousDeletedMessagesRanges = cursor.getTotalNonContiguousDeletedMessagesRange();
    subStats.nonContiguousDeletedMessagesRangesSerializedSize = cursor.getNonContiguousDeletedMessagesRangeSerializedSize();
    return subStats;
}

Example 10 with ConsumerStatsImpl

use of org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl in project pulsar by apache.

the class NonPersistentTopic method updateRates.

@Override
public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats, StatsOutputStream topicStatsStream, ClusterReplicationMetrics replStats, String namespace, boolean hydratePublishers) {
    TopicStats topicStats = threadLocalTopicStats.get();
    topicStats.reset();
    replicators.forEach((region, replicator) -> replicator.updateRates());
    nsStats.producerCount += producers.size();
    bundleStats.producerCount += producers.size();
    topicStatsStream.startObject(topic);
    topicStatsStream.startList("publishers");
    producers.values().forEach(producer -> {
        producer.updateRates();
        PublisherStatsImpl publisherStats = producer.getStats();
        topicStats.aggMsgRateIn += publisherStats.msgRateIn;
        topicStats.aggMsgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            topicStats.remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        }
        if (hydratePublishers) {
            StreamingStats.writePublisherStats(topicStatsStream, publisherStats);
        }
    });
    topicStatsStream.endList();
    // Start replicator stats
    topicStatsStream.startObject("replication");
    nsStats.replicatorCount += topicStats.remotePublishersStats.size();
    // Close replication
    topicStatsStream.endObject();
    // Start subscription stats
    topicStatsStream.startObject("subscriptions");
    nsStats.subsCount += subscriptions.size();
    subscriptions.forEach((subscriptionName, subscription) -> {
        double subMsgRateOut = 0;
        double subMsgThroughputOut = 0;
        double subMsgRateRedeliver = 0;
        // Start subscription name & consumers
        try {
            topicStatsStream.startObject(subscriptionName);
            topicStatsStream.startList("consumers");
            for (Consumer consumer : subscription.getConsumers()) {
                ++nsStats.consumerCount;
                ++bundleStats.consumerCount;
                consumer.updateRates();
                ConsumerStatsImpl consumerStats = consumer.getStats();
                subMsgRateOut += consumerStats.msgRateOut;
                subMsgThroughputOut += consumerStats.msgThroughputOut;
                subMsgRateRedeliver += consumerStats.msgRateRedeliver;
                // Populate consumer specific stats here
                StreamingStats.writeConsumerStats(topicStatsStream, subscription.getType(), consumerStats);
            }
            // Close Consumer stats
            topicStatsStream.endList();
            // Populate subscription specific stats here
            topicStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog(false));
            topicStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate());
            topicStatsStream.writePair("msgRateOut", subMsgRateOut);
            topicStatsStream.writePair("msgThroughputOut", subMsgThroughputOut);
            topicStatsStream.writePair("msgRateRedeliver", subMsgRateRedeliver);
            topicStatsStream.writePair("type", subscription.getTypeString());
            if (subscription.getDispatcher() != null) {
                subscription.getDispatcher().getMessageDropRate().calculateRate();
                topicStatsStream.writePair("msgDropRate", subscription.getDispatcher().getMessageDropRate().getValueRate());
            }
            // Close consumers
            topicStatsStream.endObject();
            topicStats.aggMsgRateOut += subMsgRateOut;
            topicStats.aggMsgThroughputOut += subMsgThroughputOut;
            nsStats.msgBacklog += subscription.getNumberOfEntriesInBacklog(false);
        } catch (Exception e) {
            log.error("Got exception when creating consumer stats for subscription {}: {}", subscriptionName, e.getMessage(), e);
        }
    });
    // Close subscription
    topicStatsStream.endObject();
    // Remaining dest stats.
    topicStats.averageMsgSize = topicStats.aggMsgRateIn == 0.0 ? 0.0 : (topicStats.aggMsgThroughputIn / topicStats.aggMsgRateIn);
    topicStatsStream.writePair("producerCount", producers.size());
    topicStatsStream.writePair("averageMsgSize", topicStats.averageMsgSize);
    topicStatsStream.writePair("msgRateIn", topicStats.aggMsgRateIn);
    topicStatsStream.writePair("msgRateOut", topicStats.aggMsgRateOut);
    topicStatsStream.writePair("msgThroughputIn", topicStats.aggMsgThroughputIn);
    topicStatsStream.writePair("msgThroughputOut", topicStats.aggMsgThroughputOut);
    topicStatsStream.writePair("msgInCount", getMsgInCounter());
    topicStatsStream.writePair("bytesInCount", getBytesInCounter());
    topicStatsStream.writePair("msgOutCount", getMsgOutCounter());
    topicStatsStream.writePair("bytesOutCount", getBytesOutCounter());
    nsStats.msgRateIn += topicStats.aggMsgRateIn;
    nsStats.msgRateOut += topicStats.aggMsgRateOut;
    nsStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    nsStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    bundleStats.msgRateIn += topicStats.aggMsgRateIn;
    bundleStats.msgRateOut += topicStats.aggMsgRateOut;
    bundleStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    bundleStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    // add publish-latency metrics
    this.addEntryLatencyStatsUsec.refresh();
    NamespaceStats.add(this.addEntryLatencyStatsUsec.getBuckets(), nsStats.addLatencyBucket);
    this.addEntryLatencyStatsUsec.reset();
    // Close topic object
    topicStatsStream.endObject();
}