Search in sources :

Example 1 with PublisherStatsImpl

use of org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl in project pulsar by apache.

the class NonPersistentTopic method asyncGetStats.

@Override
public CompletableFuture<NonPersistentTopicStatsImpl> asyncGetStats(boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) {
    CompletableFuture<NonPersistentTopicStatsImpl> future = new CompletableFuture<>();
    NonPersistentTopicStatsImpl stats = new NonPersistentTopicStatsImpl();
    ObjectObjectHashMap<String, PublisherStatsImpl> remotePublishersStats = new ObjectObjectHashMap<>();
    producers.values().forEach(producer -> {
        NonPersistentPublisherStatsImpl publisherStats = (NonPersistentPublisherStatsImpl) producer.getStats();
        stats.msgRateIn += publisherStats.msgRateIn;
        stats.msgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        } else {
            stats.addPublisher(publisherStats);
        }
    });
    stats.averageMsgSize = stats.msgRateIn == 0.0 ? 0.0 : (stats.msgThroughputIn / stats.msgRateIn);
    stats.msgInCounter = getMsgInCounter();
    stats.bytesInCounter = getBytesInCounter();
    stats.waitingPublishers = getWaitingProducersCount();
    stats.bytesOutCounter = bytesOutFromRemovedSubscriptions.longValue();
    stats.msgOutCounter = msgOutFromRemovedSubscriptions.longValue();
    subscriptions.forEach((name, subscription) -> {
        NonPersistentSubscriptionStatsImpl subStats = subscription.getStats();
        stats.msgRateOut += subStats.msgRateOut;
        stats.msgThroughputOut += subStats.msgThroughputOut;
        stats.bytesOutCounter += subStats.bytesOutCounter;
        stats.msgOutCounter += subStats.msgOutCounter;
        stats.getSubscriptions().put(name, subStats);
    });
    replicators.forEach((cluster, replicator) -> {
        NonPersistentReplicatorStatsImpl replicatorStats = replicator.getStats();
        // Add incoming msg rates
        PublisherStatsImpl pubStats = remotePublishersStats.get(replicator.getRemoteCluster());
        if (pubStats != null) {
            replicatorStats.msgRateIn = pubStats.msgRateIn;
            replicatorStats.msgThroughputIn = pubStats.msgThroughputIn;
            replicatorStats.inboundConnection = pubStats.getAddress();
            replicatorStats.inboundConnectedSince = pubStats.getConnectedSince();
        }
        stats.msgRateOut += replicatorStats.msgRateOut;
        stats.msgThroughputOut += replicatorStats.msgThroughputOut;
        stats.getReplication().put(replicator.getRemoteCluster(), replicatorStats);
    });
    stats.topicEpoch = topicEpoch.orElse(null);
    future.complete(stats);
    return future;
}
Also used : NonPersistentTopicStatsImpl(org.apache.pulsar.common.policies.data.stats.NonPersistentTopicStatsImpl) CompletableFuture(java.util.concurrent.CompletableFuture) NonPersistentReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.NonPersistentReplicatorStatsImpl) NonPersistentPublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.NonPersistentPublisherStatsImpl) NonPersistentSubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.NonPersistentSubscriptionStatsImpl) NonPersistentPublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.NonPersistentPublisherStatsImpl) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap)

Example 2 with PublisherStatsImpl

use of org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl in project pulsar by apache.

the class PersistentTopicStatsTest method testPersistentTopicStatsAggregationPartialProducerIsNotSupported.

@Test
public void testPersistentTopicStatsAggregationPartialProducerIsNotSupported() {
    TopicStatsImpl topicStats1 = new TopicStatsImpl();
    topicStats1.msgRateIn = 1;
    topicStats1.msgThroughputIn = 1;
    topicStats1.msgRateOut = 1;
    topicStats1.msgThroughputOut = 1;
    topicStats1.averageMsgSize = 1;
    topicStats1.storageSize = 1;
    final PublisherStatsImpl publisherStats1 = new PublisherStatsImpl();
    publisherStats1.setSupportsPartialProducer(false);
    publisherStats1.setProducerName("name1");
    topicStats1.addPublisher(publisherStats1);
    topicStats1.subscriptions.put("test_ns", new SubscriptionStatsImpl());
    topicStats1.replication.put("test_ns", new ReplicatorStatsImpl());
    TopicStatsImpl topicStats2 = new TopicStatsImpl();
    topicStats2.msgRateIn = 1;
    topicStats2.msgThroughputIn = 2;
    topicStats2.msgRateOut = 3;
    topicStats2.msgThroughputOut = 4;
    topicStats2.averageMsgSize = 5;
    topicStats2.storageSize = 6;
    final PublisherStatsImpl publisherStats2 = new PublisherStatsImpl();
    publisherStats2.setSupportsPartialProducer(false);
    publisherStats2.setProducerName("name1");
    topicStats2.addPublisher(publisherStats2);
    topicStats2.subscriptions.put("test_ns", new SubscriptionStatsImpl());
    topicStats2.replication.put("test_ns", new ReplicatorStatsImpl());
    TopicStatsImpl target = new TopicStatsImpl();
    target.add(topicStats1);
    target.add(topicStats2);
    assertEquals(target.msgRateIn, 2.0);
    assertEquals(target.msgThroughputIn, 3.0);
    assertEquals(target.msgRateOut, 4.0);
    assertEquals(target.msgThroughputOut, 5.0);
    assertEquals(target.averageMsgSize, 3.0);
    assertEquals(target.storageSize, 7);
    assertEquals(target.getPublishers().size(), 1);
    assertEquals(target.subscriptions.size(), 1);
    assertEquals(target.replication.size(), 1);
}
Also used : TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) Test(org.testng.annotations.Test)

Example 3 with PublisherStatsImpl

use of org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl in project pulsar by apache.

the class PersistentTopicStatsTest method testPersistentTopicStatsAggregationByProducerName.

@Test
public void testPersistentTopicStatsAggregationByProducerName() {
    TopicStatsImpl topicStats1 = new TopicStatsImpl();
    topicStats1.msgRateIn = 1;
    topicStats1.msgThroughputIn = 1;
    topicStats1.msgRateOut = 1;
    topicStats1.msgThroughputOut = 1;
    topicStats1.averageMsgSize = 1;
    topicStats1.storageSize = 1;
    final PublisherStatsImpl publisherStats1 = new PublisherStatsImpl();
    publisherStats1.setSupportsPartialProducer(true);
    publisherStats1.msgRateIn = 1;
    publisherStats1.setProducerName("name1");
    topicStats1.addPublisher(publisherStats1);
    topicStats1.subscriptions.put("test_ns", new SubscriptionStatsImpl());
    topicStats1.replication.put("test_ns", new ReplicatorStatsImpl());
    TopicStatsImpl topicStats2 = new TopicStatsImpl();
    topicStats2.msgRateIn = 1;
    topicStats2.msgThroughputIn = 2;
    topicStats2.msgRateOut = 3;
    topicStats2.msgThroughputOut = 4;
    topicStats2.averageMsgSize = 5;
    topicStats2.storageSize = 6;
    final PublisherStatsImpl publisherStats2 = new PublisherStatsImpl();
    publisherStats2.setSupportsPartialProducer(true);
    publisherStats2.msgRateIn = 1;
    publisherStats2.setProducerName("name1");
    topicStats2.addPublisher(publisherStats2);
    topicStats2.subscriptions.put("test_ns", new SubscriptionStatsImpl());
    topicStats2.replication.put("test_ns", new ReplicatorStatsImpl());
    TopicStatsImpl topicStats3 = new TopicStatsImpl();
    topicStats3.msgRateIn = 0;
    topicStats3.msgThroughputIn = 0;
    topicStats3.msgRateOut = 0;
    topicStats3.msgThroughputOut = 0;
    topicStats3.averageMsgSize = 0;
    topicStats3.storageSize = 0;
    final PublisherStatsImpl publisherStats3 = new PublisherStatsImpl();
    publisherStats3.setSupportsPartialProducer(true);
    publisherStats3.msgRateIn = 1;
    publisherStats3.setProducerName("name2");
    topicStats3.addPublisher(publisherStats3);
    topicStats3.subscriptions.put("test_ns", new SubscriptionStatsImpl());
    topicStats3.replication.put("test_ns", new ReplicatorStatsImpl());
    TopicStatsImpl target = new TopicStatsImpl();
    target.add(topicStats1);
    target.add(topicStats2);
    target.add(topicStats3);
    assertEquals(target.msgRateIn, 2.0);
    assertEquals(target.msgThroughputIn, 3.0);
    assertEquals(target.msgRateOut, 4.0);
    assertEquals(target.msgThroughputOut, 5.0);
    assertEquals(target.averageMsgSize, 2.0);
    assertEquals(target.storageSize, 7);
    assertEquals(target.getPublishers().size(), 2);
    final Map<String, Double> expectedPublishersMap = Maps.newHashMap();
    expectedPublishersMap.put("name1", 2.0);
    expectedPublishersMap.put("name2", 1.0);
    assertEquals(target.getPublishers().stream().collect(Collectors.toMap(PublisherStats::getProducerName, e -> ((PublisherStatsImpl) e).msgRateIn)), expectedPublishersMap);
    assertEquals(target.subscriptions.size(), 1);
    assertEquals(target.replication.size(), 1);
}
Also used : TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) Test(org.testng.annotations.Test)

Example 4 with PublisherStatsImpl

use of org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl in project pulsar by yahoo.

the class PersistentTopic method updateRates.

@Override
public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats, StatsOutputStream topicStatsStream, ClusterReplicationMetrics replStats, String namespace, boolean hydratePublishers) {
    this.publishRateLimitedTimes = 0;
    TopicStatsHelper topicStatsHelper = threadLocalTopicStats.get();
    topicStatsHelper.reset();
    replicators.forEach((region, replicator) -> replicator.updateRates());
    nsStats.producerCount += producers.size();
    bundleStats.producerCount += producers.size();
    topicStatsStream.startObject(topic);
    // start publisher stats
    topicStatsStream.startList("publishers");
    producers.values().forEach(producer -> {
        producer.updateRates();
        PublisherStatsImpl publisherStats = producer.getStats();
        topicStatsHelper.aggMsgRateIn += publisherStats.msgRateIn;
        topicStatsHelper.aggMsgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            topicStatsHelper.remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        }
        // Populate consumer specific stats here
        if (hydratePublishers) {
            StreamingStats.writePublisherStats(topicStatsStream, publisherStats);
        }
    });
    topicStatsStream.endList();
    // if publish-rate increases (eg: 0 to 1K) then pick max publish-rate and if publish-rate decreases then keep
    // average rate.
    lastUpdatedAvgPublishRateInMsg = topicStatsHelper.aggMsgRateIn > lastUpdatedAvgPublishRateInMsg ? topicStatsHelper.aggMsgRateIn : (topicStatsHelper.aggMsgRateIn + lastUpdatedAvgPublishRateInMsg) / 2;
    lastUpdatedAvgPublishRateInByte = topicStatsHelper.aggMsgThroughputIn > lastUpdatedAvgPublishRateInByte ? topicStatsHelper.aggMsgThroughputIn : (topicStatsHelper.aggMsgThroughputIn + lastUpdatedAvgPublishRateInByte) / 2;
    // Start replicator stats
    topicStatsStream.startObject("replication");
    nsStats.replicatorCount += topicStatsHelper.remotePublishersStats.size();
    replicators.forEach((cluster, replicator) -> {
        // Update replicator cursor state
        try {
            ((PersistentReplicator) replicator).updateCursorState();
        } catch (Exception e) {
            log.warn("[{}] Failed to update cursor state ", topic, e);
        }
        // Update replicator stats
        ReplicatorStatsImpl rStat = replicator.getStats();
        // Add incoming msg rates
        PublisherStatsImpl pubStats = topicStatsHelper.remotePublishersStats.get(replicator.getRemoteCluster());
        rStat.msgRateIn = pubStats != null ? pubStats.msgRateIn : 0;
        rStat.msgThroughputIn = pubStats != null ? pubStats.msgThroughputIn : 0;
        rStat.inboundConnection = pubStats != null ? pubStats.getAddress() : null;
        rStat.inboundConnectedSince = pubStats != null ? pubStats.getConnectedSince() : null;
        topicStatsHelper.aggMsgRateOut += rStat.msgRateOut;
        topicStatsHelper.aggMsgThroughputOut += rStat.msgThroughputOut;
        // Populate replicator specific stats here
        topicStatsStream.startObject(cluster);
        topicStatsStream.writePair("connected", rStat.connected);
        topicStatsStream.writePair("msgRateExpired", rStat.msgRateExpired);
        topicStatsStream.writePair("msgRateIn", rStat.msgRateIn);
        topicStatsStream.writePair("msgRateOut", rStat.msgRateOut);
        topicStatsStream.writePair("msgThroughputIn", rStat.msgThroughputIn);
        topicStatsStream.writePair("msgThroughputOut", rStat.msgThroughputOut);
        topicStatsStream.writePair("replicationBacklog", rStat.replicationBacklog);
        topicStatsStream.writePair("replicationDelayInSeconds", rStat.replicationDelayInSeconds);
        topicStatsStream.writePair("inboundConnection", rStat.inboundConnection);
        topicStatsStream.writePair("inboundConnectedSince", rStat.inboundConnectedSince);
        topicStatsStream.writePair("outboundConnection", rStat.outboundConnection);
        topicStatsStream.writePair("outboundConnectedSince", rStat.outboundConnectedSince);
        topicStatsStream.endObject();
        nsStats.msgReplBacklog += rStat.replicationBacklog;
        if (replStats.isMetricsEnabled()) {
            String namespaceClusterKey = replStats.getKeyName(namespace, cluster);
            ReplicationMetrics replicationMetrics = replStats.get(namespaceClusterKey);
            boolean update = false;
            if (replicationMetrics == null) {
                replicationMetrics = ReplicationMetrics.get();
                update = true;
            }
            replicationMetrics.connected += rStat.connected ? 1 : 0;
            replicationMetrics.msgRateOut += rStat.msgRateOut;
            replicationMetrics.msgThroughputOut += rStat.msgThroughputOut;
            replicationMetrics.msgReplBacklog += rStat.replicationBacklog;
            if (update) {
                replStats.put(namespaceClusterKey, replicationMetrics);
            }
            // replication delay for a namespace is the max repl-delay among all the topics under this namespace
            if (rStat.replicationDelayInSeconds > replicationMetrics.maxMsgReplDelayInSeconds) {
                replicationMetrics.maxMsgReplDelayInSeconds = rStat.replicationDelayInSeconds;
            }
        }
    });
    // Close replication
    topicStatsStream.endObject();
    // Start subscription stats
    topicStatsStream.startObject("subscriptions");
    nsStats.subsCount += subscriptions.size();
    subscriptions.forEach((subscriptionName, subscription) -> {
        double subMsgRateOut = 0;
        double subMsgThroughputOut = 0;
        double subMsgRateRedeliver = 0;
        // Start subscription name & consumers
        try {
            topicStatsStream.startObject(subscriptionName);
            topicStatsStream.startList("consumers");
            for (Consumer consumer : subscription.getConsumers()) {
                ++nsStats.consumerCount;
                ++bundleStats.consumerCount;
                consumer.updateRates();
                ConsumerStatsImpl consumerStats = consumer.getStats();
                subMsgRateOut += consumerStats.msgRateOut;
                subMsgThroughputOut += consumerStats.msgThroughputOut;
                subMsgRateRedeliver += consumerStats.msgRateRedeliver;
                StreamingStats.writeConsumerStats(topicStatsStream, subscription.getType(), consumerStats);
            }
            // Close Consumer stats
            topicStatsStream.endList();
            // Populate subscription specific stats here
            topicStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog(true));
            topicStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate());
            topicStatsStream.writePair("msgRateOut", subMsgRateOut);
            topicStatsStream.writePair("msgThroughputOut", subMsgThroughputOut);
            topicStatsStream.writePair("msgRateRedeliver", subMsgRateRedeliver);
            topicStatsStream.writePair("numberOfEntriesSinceFirstNotAckedMessage", subscription.getNumberOfEntriesSinceFirstNotAckedMessage());
            topicStatsStream.writePair("totalNonContiguousDeletedMessagesRange", subscription.getTotalNonContiguousDeletedMessagesRange());
            topicStatsStream.writePair("type", subscription.getTypeString());
            if (Subscription.isIndividualAckMode(subscription.getType())) {
                if (subscription.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) {
                    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subscription.getDispatcher();
                    topicStatsStream.writePair("blockedSubscriptionOnUnackedMsgs", dispatcher.isBlockedDispatcherOnUnackedMsgs());
                    topicStatsStream.writePair("unackedMessages", dispatcher.getTotalUnackedMessages());
                }
            }
            // Close consumers
            topicStatsStream.endObject();
            topicStatsHelper.aggMsgRateOut += subMsgRateOut;
            topicStatsHelper.aggMsgThroughputOut += subMsgThroughputOut;
            nsStats.msgBacklog += subscription.getNumberOfEntriesInBacklog(false);
            // check stuck subscription
            if (brokerService.getPulsar().getConfig().isUnblockStuckSubscriptionEnabled()) {
                subscription.checkAndUnblockIfStuck();
            }
        } catch (Exception e) {
            log.error("Got exception when creating consumer stats for subscription {}: {}", subscriptionName, e.getMessage(), e);
        }
    });
    // Close subscription
    topicStatsStream.endObject();
    // Remaining dest stats.
    topicStatsHelper.averageMsgSize = topicStatsHelper.aggMsgRateIn == 0.0 ? 0.0 : (topicStatsHelper.aggMsgThroughputIn / topicStatsHelper.aggMsgRateIn);
    topicStatsStream.writePair("producerCount", producers.size());
    topicStatsStream.writePair("averageMsgSize", topicStatsHelper.averageMsgSize);
    topicStatsStream.writePair("msgRateIn", topicStatsHelper.aggMsgRateIn);
    topicStatsStream.writePair("msgRateOut", topicStatsHelper.aggMsgRateOut);
    topicStatsStream.writePair("msgInCount", getMsgInCounter());
    topicStatsStream.writePair("bytesInCount", getBytesInCounter());
    topicStatsStream.writePair("msgOutCount", getMsgOutCounter());
    topicStatsStream.writePair("bytesOutCount", getBytesOutCounter());
    topicStatsStream.writePair("msgThroughputIn", topicStatsHelper.aggMsgThroughputIn);
    topicStatsStream.writePair("msgThroughputOut", topicStatsHelper.aggMsgThroughputOut);
    topicStatsStream.writePair("storageSize", ledger.getTotalSize());
    topicStatsStream.writePair("backlogSize", ledger.getEstimatedBacklogSize());
    topicStatsStream.writePair("pendingAddEntriesCount", ((ManagedLedgerImpl) ledger).getPendingAddEntriesCount());
    topicStatsStream.writePair("filteredEntriesCount", getFilteredEntriesCount());
    nsStats.msgRateIn += topicStatsHelper.aggMsgRateIn;
    nsStats.msgRateOut += topicStatsHelper.aggMsgRateOut;
    nsStats.msgThroughputIn += topicStatsHelper.aggMsgThroughputIn;
    nsStats.msgThroughputOut += topicStatsHelper.aggMsgThroughputOut;
    nsStats.storageSize += ledger.getEstimatedBacklogSize();
    bundleStats.msgRateIn += topicStatsHelper.aggMsgRateIn;
    bundleStats.msgRateOut += topicStatsHelper.aggMsgRateOut;
    bundleStats.msgThroughputIn += topicStatsHelper.aggMsgThroughputIn;
    bundleStats.msgThroughputOut += topicStatsHelper.aggMsgThroughputOut;
    bundleStats.cacheSize += ((ManagedLedgerImpl) ledger).getCacheSize();
    // Close topic object
    topicStatsStream.endObject();
    // add publish-latency metrics
    this.addEntryLatencyStatsUsec.refresh();
    NamespaceStats.add(this.addEntryLatencyStatsUsec.getBuckets(), nsStats.addLatencyBucket);
    this.addEntryLatencyStatsUsec.reset();
}
Also used : ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) Consumer(org.apache.pulsar.broker.service.Consumer) ReplicationMetrics(org.apache.pulsar.broker.stats.ReplicationMetrics) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) MetadataNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException) AlreadyRunningException(org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) ManagedLedgerAlreadyClosedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException) CancellationException(java.util.concurrent.CancellationException) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) TopicTerminatedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException) ManagedLedgerTerminatedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException) TopicClosedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException) TopicBacklogQuotaExceededException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBacklogQuotaExceededException) BrokerServiceException(org.apache.pulsar.broker.service.BrokerServiceException) TopicFencedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException) ExecutionException(java.util.concurrent.ExecutionException) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) UnsupportedVersionException(org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) CompletionException(java.util.concurrent.CompletionException) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl)

Example 5 with PublisherStatsImpl

use of org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl in project pulsar by yahoo.

the class PersistentTopic method asyncGetStats.

@Override
public CompletableFuture<TopicStatsImpl> asyncGetStats(boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) {
    CompletableFuture<TopicStatsImpl> statsFuture = new CompletableFuture<>();
    TopicStatsImpl stats = new TopicStatsImpl();
    ObjectObjectHashMap<String, PublisherStatsImpl> remotePublishersStats = new ObjectObjectHashMap<>();
    producers.values().forEach(producer -> {
        PublisherStatsImpl publisherStats = producer.getStats();
        stats.msgRateIn += publisherStats.msgRateIn;
        stats.msgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        } else {
            stats.addPublisher(publisherStats);
        }
    });
    stats.averageMsgSize = stats.msgRateIn == 0.0 ? 0.0 : (stats.msgThroughputIn / stats.msgRateIn);
    stats.msgInCounter = getMsgInCounter();
    stats.bytesInCounter = getBytesInCounter();
    stats.msgChunkPublished = this.msgChunkPublished;
    stats.waitingPublishers = getWaitingProducersCount();
    stats.bytesOutCounter = bytesOutFromRemovedSubscriptions.longValue();
    stats.msgOutCounter = msgOutFromRemovedSubscriptions.longValue();
    stats.publishRateLimitedTimes = publishRateLimitedTimes;
    subscriptions.forEach((name, subscription) -> {
        SubscriptionStatsImpl subStats = subscription.getStats(getPreciseBacklog, subscriptionBacklogSize, getEarliestTimeInBacklog);
        stats.msgRateOut += subStats.msgRateOut;
        stats.msgThroughputOut += subStats.msgThroughputOut;
        stats.bytesOutCounter += subStats.bytesOutCounter;
        stats.msgOutCounter += subStats.msgOutCounter;
        stats.subscriptions.put(name, subStats);
        stats.nonContiguousDeletedMessagesRanges += subStats.nonContiguousDeletedMessagesRanges;
        stats.nonContiguousDeletedMessagesRangesSerializedSize += subStats.nonContiguousDeletedMessagesRangesSerializedSize;
    });
    replicators.forEach((cluster, replicator) -> {
        ReplicatorStatsImpl replicatorStats = replicator.getStats();
        // Add incoming msg rates
        PublisherStatsImpl pubStats = remotePublishersStats.get(replicator.getRemoteCluster());
        if (pubStats != null) {
            replicatorStats.msgRateIn = pubStats.msgRateIn;
            replicatorStats.msgThroughputIn = pubStats.msgThroughputIn;
            replicatorStats.inboundConnection = pubStats.getAddress();
            replicatorStats.inboundConnectedSince = pubStats.getConnectedSince();
        }
        stats.msgRateOut += replicatorStats.msgRateOut;
        stats.msgThroughputOut += replicatorStats.msgThroughputOut;
        stats.replication.put(replicator.getRemoteCluster(), replicatorStats);
    });
    stats.storageSize = ledger.getTotalSize();
    stats.backlogSize = ledger.getEstimatedBacklogSize();
    stats.deduplicationStatus = messageDeduplication.getStatus().toString();
    stats.topicEpoch = topicEpoch.orElse(null);
    stats.offloadedStorageSize = ledger.getOffloadedSize();
    stats.lastOffloadLedgerId = ledger.getLastOffloadedLedgerId();
    stats.lastOffloadSuccessTimeStamp = ledger.getLastOffloadedSuccessTimestamp();
    stats.lastOffloadFailureTimeStamp = ledger.getLastOffloadedFailureTimestamp();
    Optional<CompactorMXBean> mxBean = getCompactorMXBean();
    stats.compaction.reset();
    mxBean.flatMap(bean -> bean.getCompactionRecordForTopic(topic)).map(compactionRecord -> {
        stats.compaction.lastCompactionRemovedEventCount = compactionRecord.getLastCompactionRemovedEventCount();
        stats.compaction.lastCompactionSucceedTimestamp = compactionRecord.getLastCompactionSucceedTimestamp();
        stats.compaction.lastCompactionFailedTimestamp = compactionRecord.getLastCompactionFailedTimestamp();
        stats.compaction.lastCompactionDurationTimeInMills = compactionRecord.getLastCompactionDurationTimeInMills();
        return compactionRecord;
    });
    if (getEarliestTimeInBacklog && stats.backlogSize != 0) {
        ledger.getEarliestMessagePublishTimeInBacklog().whenComplete((earliestTime, e) -> {
            if (e != null) {
                log.error("[{}] Failed to get earliest message publish time in backlog", topic, e);
                statsFuture.completeExceptionally(e);
            } else {
                stats.earliestMsgPublishTimeInBacklogs = earliestTime;
                statsFuture.complete(stats);
            }
        });
    } else {
        statsFuture.complete(stats);
    }
    return statsFuture;
}
Also used : TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) NamingException(org.apache.pulsar.broker.service.BrokerServiceException.NamingException) Topic(org.apache.pulsar.broker.service.Topic) PulsarClientImpl(org.apache.pulsar.client.impl.PulsarClientImpl) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) CloseCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap) StringUtils(org.apache.commons.lang3.StringUtils) SubscribeRate(org.apache.pulsar.common.policies.data.SubscribeRate) MetadataNotFoundException(org.apache.bookkeeper.mledger.ManagedLedgerException.MetadataNotFoundException) OpenCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback) TxnID(org.apache.pulsar.client.api.transaction.TxnID) ManagedCursor(org.apache.bookkeeper.mledger.ManagedCursor) AlreadyRunningException(org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException) Map(java.util.Map) StatsOutputStream(org.apache.pulsar.utils.StatsOutputStream) EnumSet(java.util.EnumSet) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) UpdatePropertiesCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.UpdatePropertiesCallback) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) ManagedLedgerAlreadyClosedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException) CancellationException(java.util.concurrent.CancellationException) ManagedCursorContainer(org.apache.bookkeeper.mledger.impl.ManagedCursorContainer) Set(java.util.Set) BrokerService(org.apache.pulsar.broker.service.BrokerService) NamespaceStats(org.apache.pulsar.broker.stats.NamespaceStats) AddEntryCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) OffloadProcessStatus(org.apache.pulsar.client.admin.OffloadProcessStatus) BatchMessageIdImpl(org.apache.pulsar.client.impl.BatchMessageIdImpl) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) TransactionPendingAckStats(org.apache.pulsar.common.policies.data.TransactionPendingAckStats) ConcurrentOpenHashMap(org.apache.pulsar.common.util.collections.ConcurrentOpenHashMap) ManagedLedgerFencedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException) TopicTerminatedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicTerminatedException) PartitionedTopicResources(org.apache.pulsar.broker.resources.NamespaceResources.PartitionedTopicResources) ManagedLedgerTerminatedException(org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) BookieId(org.apache.bookkeeper.net.BookieId) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) TransactionInPendingAckStats(org.apache.pulsar.common.policies.data.TransactionInPendingAckStats) Subscription(org.apache.pulsar.broker.service.Subscription) LedgerInfo(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.LedgerInfo) NamespaceService(org.apache.pulsar.broker.namespace.NamespaceService) Consumer(org.apache.pulsar.broker.service.Consumer) TerminateCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.TerminateCallback) ArrayList(java.util.ArrayList) TopicClosedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicClosedException) Commands(org.apache.pulsar.common.protocol.Commands) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Lists(com.google.common.collect.Lists) CursorStats(org.apache.pulsar.common.policies.data.ManagedLedgerInternalStats.CursorStats) DateFormatter(org.apache.pulsar.common.util.DateFormatter) TopicBacklogQuotaExceededException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBacklogQuotaExceededException) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) CompactorMXBean(org.apache.pulsar.compaction.CompactorMXBean) Dispatcher(org.apache.pulsar.broker.service.Dispatcher) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) BrokerServiceException(org.apache.pulsar.broker.service.BrokerServiceException) StreamingStats(org.apache.pulsar.broker.service.StreamingStats) AbstractReplicator(org.apache.pulsar.broker.service.AbstractReplicator) TopicFencedException(org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException) ExecutionException(java.util.concurrent.ExecutionException) AtomicLong(java.util.concurrent.atomic.AtomicLong) Compactor(org.apache.pulsar.compaction.Compactor) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) Codec(org.apache.pulsar.common.util.Codec) SchemaData(org.apache.pulsar.common.protocol.schema.SchemaData) SystemTopicNames.isEventSystemTopic(org.apache.pulsar.common.naming.SystemTopicNames.isEventSystemTopic) ScheduledFuture(java.util.concurrent.ScheduledFuture) COMPACTION_SUBSCRIPTION(org.apache.pulsar.compaction.Compactor.COMPACTION_SUBSCRIPTION) BookkeeperSchemaStorage(org.apache.pulsar.broker.service.schema.BookkeeperSchemaStorage) BiFunction(java.util.function.BiFunction) UnsupportedVersionException(org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException) LoggerFactory(org.slf4j.LoggerFactory) SystemTopicNames(org.apache.pulsar.common.naming.SystemTopicNames) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) Type(org.apache.pulsar.broker.service.persistent.DispatchRateLimiter.Type) BacklogQuotaType(org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) TxnAction(org.apache.pulsar.common.api.proto.TxnAction) DeleteCursorCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCursorCallback) TransactionInBufferStats(org.apache.pulsar.common.policies.data.TransactionInBufferStats) ReplicationMetrics(org.apache.pulsar.broker.stats.ReplicationMetrics) SubscribeRateLimiter.isSubscribeRateEnabled(org.apache.pulsar.broker.service.persistent.SubscribeRateLimiter.isSubscribeRateEnabled) CompletionException(java.util.concurrent.CompletionException) SubscriptionNotFoundException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionNotFoundException) Position(org.apache.bookkeeper.mledger.Position) IndividualDeletedEntries(org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries) CompactedTopicContext(org.apache.pulsar.compaction.CompactedTopicContext) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) InactiveTopicDeleteMode(org.apache.pulsar.common.policies.data.InactiveTopicDeleteMode) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) ManagedCursorImpl(org.apache.bookkeeper.mledger.impl.ManagedCursorImpl) List(java.util.List) FutureUtil(org.apache.pulsar.common.util.FutureUtil) MLPendingAckStore(org.apache.pulsar.broker.transaction.pendingack.impl.MLPendingAckStore) Optional(java.util.Optional) NamespaceBundleStats(org.apache.pulsar.policies.data.loadbalancer.NamespaceBundleStats) LedgerMetadata(org.apache.bookkeeper.client.api.LedgerMetadata) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) LongAdder(java.util.concurrent.atomic.LongAdder) TopicName(org.apache.pulsar.common.naming.TopicName) Getter(lombok.Getter) Entry(org.apache.bookkeeper.mledger.Entry) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) CompletableFuture(java.util.concurrent.CompletableFuture) KeySharedMeta(org.apache.pulsar.common.api.proto.KeySharedMeta) CommandSubscribe(org.apache.pulsar.common.api.proto.CommandSubscribe) SubscriptionOption(org.apache.pulsar.broker.service.SubscriptionOption) ByteBuf(io.netty.buffer.ByteBuf) CompactedTopicImpl(org.apache.pulsar.compaction.CompactedTopicImpl) FastThreadLocal(io.netty.util.concurrent.FastThreadLocal) TransactionBufferDisable(org.apache.pulsar.broker.transaction.buffer.impl.TransactionBufferDisable) LongRunningProcessStatus(org.apache.pulsar.client.admin.LongRunningProcessStatus) ConsumerStatsImpl(org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl) ClusterReplicationMetrics(org.apache.pulsar.broker.stats.ClusterReplicationMetrics) AbstractTopic(org.apache.pulsar.broker.service.AbstractTopic) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) MapUtils(org.apache.commons.collections4.MapUtils) DEFAULT_CONSUMER_EPOCH(org.apache.pulsar.common.protocol.Commands.DEFAULT_CONSUMER_EPOCH) TransportCnx(org.apache.pulsar.broker.service.TransportCnx) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) Logger(org.slf4j.Logger) CompactedTopic(org.apache.pulsar.compaction.CompactedTopic) Preconditions.checkNotNull(com.google.common.base.Preconditions.checkNotNull) OffloadCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.OffloadCallback) PersistenceException(org.apache.pulsar.broker.service.BrokerServiceException.PersistenceException) Producer(org.apache.pulsar.broker.service.Producer) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) Maps(com.google.common.collect.Maps) TransactionBufferStats(org.apache.pulsar.common.policies.data.TransactionBufferStats) TransactionBuffer(org.apache.pulsar.broker.transaction.buffer.TransactionBuffer) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) MessageId(org.apache.pulsar.client.api.MessageId) StringUtils.isBlank(org.apache.commons.lang3.StringUtils.isBlank) Clock(java.time.Clock) ConsumerBusyException(org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException) Replicator(org.apache.pulsar.broker.service.Replicator) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Collections(java.util.Collections) CompletableFuture(java.util.concurrent.CompletableFuture) SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) CompactorMXBean(org.apache.pulsar.compaction.CompactorMXBean) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) ObjectObjectHashMap(com.carrotsearch.hppc.ObjectObjectHashMap)

Aggregations

PublisherStatsImpl (org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl)35 Test (org.testng.annotations.Test)23 ReplicatorStatsImpl (org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl)21 SubscriptionStatsImpl (org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl)18 TopicStatsImpl (org.apache.pulsar.common.policies.data.stats.TopicStatsImpl)17 ExecutionException (java.util.concurrent.ExecutionException)9 PulsarServerException (org.apache.pulsar.broker.PulsarServerException)9 BrokerServiceException (org.apache.pulsar.broker.service.BrokerServiceException)9 ConsumerBusyException (org.apache.pulsar.broker.service.BrokerServiceException.ConsumerBusyException)9 NamingException (org.apache.pulsar.broker.service.BrokerServiceException.NamingException)9 NotAllowedException (org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException)9 SubscriptionBusyException (org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException)9 TopicBusyException (org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException)9 TopicFencedException (org.apache.pulsar.broker.service.BrokerServiceException.TopicFencedException)9 UnsupportedVersionException (org.apache.pulsar.broker.service.BrokerServiceException.UnsupportedVersionException)9 Consumer (org.apache.pulsar.broker.service.Consumer)9 ConsumerStatsImpl (org.apache.pulsar.common.policies.data.stats.ConsumerStatsImpl)9 MetadataStoreException (org.apache.pulsar.metadata.api.MetadataStoreException)9 ObjectObjectHashMap (com.carrotsearch.hppc.ObjectObjectHashMap)6 CancellationException (java.util.concurrent.CancellationException)6