Examples with NonPersistentSubscriptionStatsImpl org.apache.pulsar.common.policies.data.stats.NonPersistentSubscriptionStatsImpl used on opensource projects

Example 1 with NonPersistentSubscriptionStatsImpl

use of org.apache.pulsar.common.policies.data.stats.NonPersistentSubscriptionStatsImpl in project pulsar by apache.

the class NonPersistentTopic method asyncGetStats.

@Override
public CompletableFuture<NonPersistentTopicStatsImpl> asyncGetStats(boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) {
    CompletableFuture<NonPersistentTopicStatsImpl> future = new CompletableFuture<>();
    NonPersistentTopicStatsImpl stats = new NonPersistentTopicStatsImpl();
    ObjectObjectHashMap<String, PublisherStatsImpl> remotePublishersStats = new ObjectObjectHashMap<>();
    producers.values().forEach(producer -> {
        NonPersistentPublisherStatsImpl publisherStats = (NonPersistentPublisherStatsImpl) producer.getStats();
        stats.msgRateIn += publisherStats.msgRateIn;
        stats.msgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        } else {
            stats.addPublisher(publisherStats);
        }
    });
    stats.averageMsgSize = stats.msgRateIn == 0.0 ? 0.0 : (stats.msgThroughputIn / stats.msgRateIn);
    stats.msgInCounter = getMsgInCounter();
    stats.bytesInCounter = getBytesInCounter();
    stats.waitingPublishers = getWaitingProducersCount();
    stats.bytesOutCounter = bytesOutFromRemovedSubscriptions.longValue();
    stats.msgOutCounter = msgOutFromRemovedSubscriptions.longValue();
    subscriptions.forEach((name, subscription) -> {
        NonPersistentSubscriptionStatsImpl subStats = subscription.getStats();
        stats.msgRateOut += subStats.msgRateOut;
        stats.msgThroughputOut += subStats.msgThroughputOut;
        stats.bytesOutCounter += subStats.bytesOutCounter;
        stats.msgOutCounter += subStats.msgOutCounter;
        stats.getSubscriptions().put(name, subStats);
    });
    replicators.forEach((cluster, replicator) -> {
        NonPersistentReplicatorStatsImpl replicatorStats = replicator.getStats();
        // Add incoming msg rates
        PublisherStatsImpl pubStats = remotePublishersStats.get(replicator.getRemoteCluster());
        if (pubStats != null) {
            replicatorStats.msgRateIn = pubStats.msgRateIn;
            replicatorStats.msgThroughputIn = pubStats.msgThroughputIn;
            replicatorStats.inboundConnection = pubStats.getAddress();
            replicatorStats.inboundConnectedSince = pubStats.getConnectedSince();
        }
        stats.msgRateOut += replicatorStats.msgRateOut;
        stats.msgThroughputOut += replicatorStats.msgThroughputOut;
        stats.getReplication().put(replicator.getRemoteCluster(), replicatorStats);
    });
    stats.topicEpoch = topicEpoch.orElse(null);
    future.complete(stats);
    return future;
}

Example 2 with NonPersistentSubscriptionStatsImpl

use of org.apache.pulsar.common.policies.data.stats.NonPersistentSubscriptionStatsImpl in project pulsar by apache.

the class NonPersistentSubscription method getStats.

public NonPersistentSubscriptionStatsImpl getStats() {
    NonPersistentSubscriptionStatsImpl subStats = new NonPersistentSubscriptionStatsImpl();
    subStats.bytesOutCounter = bytesOutFromRemovedConsumers.longValue();
    subStats.msgOutCounter = msgOutFromRemovedConsumer.longValue();
    NonPersistentDispatcher dispatcher = this.dispatcher;
    if (dispatcher != null) {
        dispatcher.getConsumers().forEach(consumer -> {
            ConsumerStatsImpl consumerStats = consumer.getStats();
            subStats.consumers.add(consumerStats);
            subStats.msgRateOut += consumerStats.msgRateOut;
            subStats.msgThroughputOut += consumerStats.msgThroughputOut;
            subStats.bytesOutCounter += consumerStats.bytesOutCounter;
            subStats.msgOutCounter += consumerStats.msgOutCounter;
            subStats.msgRateRedeliver += consumerStats.msgRateRedeliver;
        });
    }
    subStats.type = getTypeString();
    subStats.msgDropRate = dispatcher.getMessageDropRate().getValueRate();
    KeySharedMode keySharedMode = this.keySharedMode;
    if (getType() == SubType.Key_Shared && keySharedMode != null) {
        subStats.keySharedMode = keySharedMode.toString();
    }
    return subStats;
}

Example 3 with NonPersistentSubscriptionStatsImpl

use of org.apache.pulsar.common.policies.data.stats.NonPersistentSubscriptionStatsImpl in project pulsar by apache.

the class NonPersistentPartitionedTopicStatsTest method testPartitionedTopicStats.

@Test
public void testPartitionedTopicStats() {
    NonPersistentPartitionedTopicStatsImpl nonPersistentPartitionedTopicStats = new NonPersistentPartitionedTopicStatsImpl();
    nonPersistentPartitionedTopicStats.msgRateIn = 1;
    nonPersistentPartitionedTopicStats.msgThroughputIn = 1;
    nonPersistentPartitionedTopicStats.msgRateOut = 1;
    nonPersistentPartitionedTopicStats.msgThroughputOut = 1;
    nonPersistentPartitionedTopicStats.averageMsgSize = 1;
    nonPersistentPartitionedTopicStats.storageSize = 1;
    nonPersistentPartitionedTopicStats.getPublishers().add(new NonPersistentPublisherStatsImpl());
    nonPersistentPartitionedTopicStats.getSubscriptions().put("test_ns", new NonPersistentSubscriptionStatsImpl());
    nonPersistentPartitionedTopicStats.getReplication().put("test_ns", new NonPersistentReplicatorStatsImpl());
    nonPersistentPartitionedTopicStats.metadata.partitions = 1;
    nonPersistentPartitionedTopicStats.partitions.put("test", nonPersistentPartitionedTopicStats);
    nonPersistentPartitionedTopicStats.reset();
    assertEquals(nonPersistentPartitionedTopicStats.msgRateIn, 0.0);
    assertEquals(nonPersistentPartitionedTopicStats.msgThroughputIn, 0.0);
    assertEquals(nonPersistentPartitionedTopicStats.msgRateOut, 0.0);
    assertEquals(nonPersistentPartitionedTopicStats.msgThroughputOut, 0.0);
    assertEquals(nonPersistentPartitionedTopicStats.averageMsgSize, 0.0);
    assertEquals(nonPersistentPartitionedTopicStats.storageSize, 0);
    assertEquals(nonPersistentPartitionedTopicStats.getPublishers().size(), 0);
    assertEquals(nonPersistentPartitionedTopicStats.getSubscriptions().size(), 0);
    assertEquals(nonPersistentPartitionedTopicStats.getReplication().size(), 0);
    assertEquals(nonPersistentPartitionedTopicStats.metadata.partitions, 0);
    assertEquals(nonPersistentPartitionedTopicStats.partitions.size(), 0);
}

Example 4 with NonPersistentSubscriptionStatsImpl

use of org.apache.pulsar.common.policies.data.stats.NonPersistentSubscriptionStatsImpl in project pulsar by apache.

the class NamespaceStatsAggregator method getTopicStats.

private static void getTopicStats(Topic topic, TopicStats stats, boolean includeConsumerMetrics, boolean includeProducerMetrics, boolean getPreciseBacklog, boolean subscriptionBacklogSize, Optional<CompactorMXBean> compactorMXBean) {
    stats.reset();
    if (topic instanceof PersistentTopic) {
        // Managed Ledger stats
        ManagedLedger ml = ((PersistentTopic) topic).getManagedLedger();
        ManagedLedgerMBeanImpl mlStats = (ManagedLedgerMBeanImpl) ml.getStats();
        stats.managedLedgerStats.storageSize = mlStats.getStoredMessagesSize();
        stats.managedLedgerStats.storageLogicalSize = mlStats.getStoredMessagesLogicalSize();
        stats.managedLedgerStats.backlogSize = ml.getEstimatedBacklogSize();
        stats.managedLedgerStats.offloadedStorageUsed = ml.getOffloadedSize();
        stats.backlogQuotaLimit = topic.getBacklogQuota(BacklogQuota.BacklogQuotaType.destination_storage).getLimitSize();
        stats.backlogQuotaLimitTime = topic.getBacklogQuota(BacklogQuota.BacklogQuotaType.message_age).getLimitTime();
        stats.managedLedgerStats.storageWriteLatencyBuckets.addAll(mlStats.getInternalAddEntryLatencyBuckets());
        stats.managedLedgerStats.storageWriteLatencyBuckets.refresh();
        stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.addAll(mlStats.getInternalLedgerAddEntryLatencyBuckets());
        stats.managedLedgerStats.storageLedgerWriteLatencyBuckets.refresh();
        stats.managedLedgerStats.entrySizeBuckets.addAll(mlStats.getInternalEntrySizeBuckets());
        stats.managedLedgerStats.entrySizeBuckets.refresh();
        stats.managedLedgerStats.storageWriteRate = mlStats.getAddEntryMessagesRate();
        stats.managedLedgerStats.storageReadRate = mlStats.getReadEntriesRate();
    }
    TopicStatsImpl tStatus = topic.getStats(getPreciseBacklog, subscriptionBacklogSize, false);
    stats.msgInCounter = tStatus.msgInCounter;
    stats.bytesInCounter = tStatus.bytesInCounter;
    stats.msgOutCounter = tStatus.msgOutCounter;
    stats.bytesOutCounter = tStatus.bytesOutCounter;
    stats.averageMsgSize = tStatus.averageMsgSize;
    stats.publishRateLimitedTimes = tStatus.publishRateLimitedTimes;
    stats.producersCount = 0;
    topic.getProducers().values().forEach(producer -> {
        if (producer.isRemote()) {
            AggregatedReplicationStats replStats = stats.replicationStats.computeIfAbsent(producer.getRemoteCluster(), k -> new AggregatedReplicationStats());
            replStats.msgRateIn += producer.getStats().msgRateIn;
            replStats.msgThroughputIn += producer.getStats().msgThroughputIn;
        } else {
            // Local producer
            stats.producersCount++;
            stats.rateIn += producer.getStats().msgRateIn;
            stats.throughputIn += producer.getStats().msgThroughputIn;
            if (includeProducerMetrics) {
                AggregatedProducerStats producerStats = stats.producerStats.computeIfAbsent(producer.getProducerName(), k -> new AggregatedProducerStats());
                producerStats.producerId = producer.getStats().producerId;
                producerStats.msgRateIn = producer.getStats().msgRateIn;
                producerStats.msgThroughputIn = producer.getStats().msgThroughputIn;
                producerStats.averageMsgSize = producer.getStats().averageMsgSize;
            }
        }
    });
    if (topic instanceof PersistentTopic) {
        tStatus.subscriptions.forEach((subName, subscriptionStats) -> {
            AggregatedSubscriptionStats subsStats = stats.subscriptionStats.computeIfAbsent(subName, k -> new AggregatedSubscriptionStats());
            aggregateTopicStats(stats, subscriptionStats, subsStats);
        });
    } else {
        ((NonPersistentTopicStatsImpl) tStatus).getNonPersistentSubscriptions().forEach((subName, nonPersistentSubscriptionStats) -> {
            NonPersistentSubscriptionStatsImpl subscriptionStats = (NonPersistentSubscriptionStatsImpl) nonPersistentSubscriptionStats;
            AggregatedSubscriptionStats subsStats = stats.subscriptionStats.computeIfAbsent(subName, k -> new AggregatedSubscriptionStats());
            aggregateTopicStats(stats, subscriptionStats, subsStats);
            subsStats.msgDropRate += subscriptionStats.getMsgDropRate();
        });
    }
    // Consumer stats can be a lot if a subscription has many consumers
    if (includeConsumerMetrics) {
        topic.getSubscriptions().forEach((name, subscription) -> {
            AggregatedSubscriptionStats subsStats = stats.subscriptionStats.computeIfAbsent(name, k -> new AggregatedSubscriptionStats());
            subscription.getConsumers().forEach(consumer -> {
                ConsumerStatsImpl conStats = consumer.getStats();
                AggregatedConsumerStats consumerStats = subsStats.consumerStat.computeIfAbsent(consumer, k -> new AggregatedConsumerStats());
                consumerStats.unackedMessages = conStats.unackedMessages;
                consumerStats.msgRateRedeliver = conStats.msgRateRedeliver;
                consumerStats.msgRateOut = conStats.msgRateOut;
                consumerStats.msgThroughputOut = conStats.msgThroughputOut;
                consumerStats.bytesOutCounter = conStats.bytesOutCounter;
                consumerStats.msgOutCounter = conStats.msgOutCounter;
                consumerStats.availablePermits = conStats.availablePermits;
                consumerStats.blockedSubscriptionOnUnackedMsgs = conStats.blockedConsumerOnUnackedMsgs;
            });
        });
    }
    topic.getReplicators().forEach((cluster, replicator) -> {
        AggregatedReplicationStats aggReplStats = stats.replicationStats.computeIfAbsent(cluster, k -> new AggregatedReplicationStats());
        ReplicatorStatsImpl replStats = replicator.getStats();
        aggReplStats.msgRateOut += replStats.msgRateOut;
        aggReplStats.msgThroughputOut += replStats.msgThroughputOut;
        aggReplStats.replicationBacklog += replStats.replicationBacklog;
        aggReplStats.msgRateIn += replStats.msgRateIn;
        aggReplStats.msgThroughputIn += replStats.msgThroughputIn;
        aggReplStats.msgRateExpired += replStats.msgRateExpired;
        aggReplStats.connectedCount += replStats.connected ? 1 : 0;
        aggReplStats.replicationDelayInSeconds += replStats.replicationDelayInSeconds;
    });
    compactorMXBean.flatMap(mxBean -> mxBean.getCompactionRecordForTopic(topic.getName())).map(compactionRecord -> {
        stats.compactionRemovedEventCount = compactionRecord.getCompactionRemovedEventCount();
        stats.compactionSucceedCount = compactionRecord.getCompactionSucceedCount();
        stats.compactionFailedCount = compactionRecord.getCompactionFailedCount();
        stats.compactionDurationTimeInMills = compactionRecord.getCompactionDurationTimeInMills();
        stats.compactionReadThroughput = compactionRecord.getCompactionReadThroughput();
        stats.compactionWriteThroughput = compactionRecord.getCompactionWriteThroughput();
        stats.compactionLatencyBuckets.addAll(compactionRecord.getCompactionLatencyStats());
        stats.compactionLatencyBuckets.refresh();
        PersistentTopic persistentTopic = (PersistentTopic) topic;
        Optional<CompactedTopicContext> compactedTopicContext = persistentTopic.getCompactedTopicContext();
        if (compactedTopicContext.isPresent()) {
            LedgerHandle ledger = compactedTopicContext.get().getLedger();
            long entries = ledger.getLastAddConfirmed() + 1;
            long size = ledger.getLength();
            stats.compactionCompactedEntriesCount = entries;
            stats.compactionCompactedEntriesSize = size;
        }
        return compactionRecord;
    });
}