Examples with ConsumerStats org.apache.pulsar.common.policies.data.ConsumerStats used on opensource projects

Example 1 with ConsumerStats

use of org.apache.pulsar.common.policies.data.ConsumerStats in project incubator-pulsar by apache.

the class MembershipManager method getCurrentMembership.

public List<WorkerInfo> getCurrentMembership() {
    List<WorkerInfo> workerIds = new LinkedList<>();
    PersistentTopicStats persistentTopicStats = null;
    PulsarAdmin pulsarAdmin = this.getPulsarAdminClient();
    try {
        persistentTopicStats = pulsarAdmin.persistentTopics().getStats(this.workerConfig.getClusterCoordinationTopic());
    } catch (PulsarAdminException e) {
        log.error("Failed to get status of coordinate topic {}", this.workerConfig.getClusterCoordinationTopic(), e);
        throw new RuntimeException(e);
    }
    for (ConsumerStats consumerStats : persistentTopicStats.subscriptions.get(COORDINATION_TOPIC_SUBSCRIPTION).consumers) {
        WorkerInfo workerInfo = WorkerInfo.parseFrom(consumerStats.metadata.get(WORKER_IDENTIFIER));
        workerIds.add(workerInfo);
    }
    return workerIds;
}

Example 2 with ConsumerStats

use of org.apache.pulsar.common.policies.data.ConsumerStats in project incubator-pulsar by apache.

the class ServerCnx method createConsumerStatsResponse.

CommandConsumerStatsResponse.Builder createConsumerStatsResponse(Consumer consumer, long requestId) {
    CommandConsumerStatsResponse.Builder commandConsumerStatsResponseBuilder = CommandConsumerStatsResponse.newBuilder();
    ConsumerStats consumerStats = consumer.getStats();
    commandConsumerStatsResponseBuilder.setRequestId(requestId);
    commandConsumerStatsResponseBuilder.setMsgRateOut(consumerStats.msgRateOut);
    commandConsumerStatsResponseBuilder.setMsgThroughputOut(consumerStats.msgThroughputOut);
    commandConsumerStatsResponseBuilder.setMsgRateRedeliver(consumerStats.msgRateRedeliver);
    commandConsumerStatsResponseBuilder.setConsumerName(consumerStats.consumerName);
    commandConsumerStatsResponseBuilder.setAvailablePermits(consumerStats.availablePermits);
    commandConsumerStatsResponseBuilder.setUnackedMessages(consumerStats.unackedMessages);
    commandConsumerStatsResponseBuilder.setBlockedConsumerOnUnackedMsgs(consumerStats.blockedConsumerOnUnackedMsgs);
    commandConsumerStatsResponseBuilder.setAddress(consumerStats.address);
    commandConsumerStatsResponseBuilder.setConnectedSince(consumerStats.connectedSince);
    Subscription subscription = consumer.getSubscription();
    commandConsumerStatsResponseBuilder.setMsgBacklog(subscription.getNumberOfEntriesInBacklog());
    commandConsumerStatsResponseBuilder.setMsgRateExpired(subscription.getExpiredMessageRate());
    commandConsumerStatsResponseBuilder.setType(subscription.getTypeString());
    return commandConsumerStatsResponseBuilder;
}

Example 3 with ConsumerStats

use of org.apache.pulsar.common.policies.data.ConsumerStats in project incubator-pulsar by apache.

the class PersistentQueueE2ETest method testUnackedCountWithRedeliveries.

@Test
public void testUnackedCountWithRedeliveries() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/testUnackedCountWithRedeliveries";
    final String subName = "sub3";
    final int numMsgs = 10;
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).receiverQueueSize(10).subscriptionType(SubscriptionType.Shared);
    ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
    for (int i = 0; i < numMsgs; i++) {
        producer.send(("hello-" + i).getBytes());
    }
    Set<MessageId> c1_receivedMessages = new HashSet<>();
    // C-1 gets all messages but doesn't ack
    for (int i = 0; i < numMsgs; i++) {
        c1_receivedMessages.add(consumer1.receive().getMessageId());
    }
    // C-2 will not get any message initially, since everything went to C-1 already
    Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
    // Trigger C-1 to redeliver everything, half will go C-1 again and the other half to C-2
    consumer1.redeliverUnacknowledgedMessages(c1_receivedMessages);
    // Consumer 2 will also receive all message but not ack
    for (int i = 0; i < numMsgs; i++) {
        consumer2.receive();
    }
    for (MessageId msgId : c1_receivedMessages) {
        consumer1.acknowledge(msgId);
    }
    PersistentTopicStats stats = admin.persistentTopics().getStats(topicName);
    // Unacked messages count should be 0 for both consumers at this point
    SubscriptionStats subStats = stats.subscriptions.get(subName);
    assertEquals(subStats.msgBacklog, 0);
    for (ConsumerStats cs : subStats.consumers) {
        assertEquals(cs.unackedMessages, 0);
    }
    producer.close();
    consumer1.close();
    consumer2.close();
    admin.persistentTopics().delete(topicName);
}

Example 4 with ConsumerStats

use of org.apache.pulsar.common.policies.data.ConsumerStats in project incubator-pulsar by apache.

the class PersistentSubscription method getStats.

public SubscriptionStats getStats() {
    SubscriptionStats subStats = new SubscriptionStats();
    Dispatcher dispatcher = this.dispatcher;
    if (dispatcher != null) {
        dispatcher.getConsumers().forEach(consumer -> {
            ConsumerStats consumerStats = consumer.getStats();
            subStats.consumers.add(consumerStats);
            subStats.msgRateOut += consumerStats.msgRateOut;
            subStats.msgThroughputOut += consumerStats.msgThroughputOut;
            subStats.msgRateRedeliver += consumerStats.msgRateRedeliver;
            subStats.unackedMessages += consumerStats.unackedMessages;
        });
    }
    subStats.type = getType();
    if (SubType.Shared.equals(subStats.type)) {
        if (dispatcher instanceof PersistentDispatcherMultipleConsumers) {
            subStats.unackedMessages = ((PersistentDispatcherMultipleConsumers) dispatcher).getTotalUnackedMessages();
            subStats.blockedSubscriptionOnUnackedMsgs = ((PersistentDispatcherMultipleConsumers) dispatcher).isBlockedDispatcherOnUnackedMsgs();
        }
    }
    subStats.msgBacklog = getNumberOfEntriesInBacklog();
    subStats.msgRateExpired = expiryMonitor.getMessageExpiryRate();
    return subStats;
}

Example 5 with ConsumerStats

use of org.apache.pulsar.common.policies.data.ConsumerStats in project incubator-pulsar by apache.

the class PersistentTopic method updateRates.

public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats, StatsOutputStream topicStatsStream, ClusterReplicationMetrics replStats, String namespace) {
    TopicStats topicStats = threadLocalTopicStats.get();
    topicStats.reset();
    replicators.forEach((region, replicator) -> replicator.updateRates());
    nsStats.producerCount += producers.size();
    bundleStats.producerCount += producers.size();
    topicStatsStream.startObject(topic);
    producers.forEach(producer -> {
        producer.updateRates();
        PublisherStats publisherStats = producer.getStats();
        topicStats.aggMsgRateIn += publisherStats.msgRateIn;
        topicStats.aggMsgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            topicStats.remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        }
    });
    // Creating publishers object for backward compatibility
    topicStatsStream.startList("publishers");
    topicStatsStream.endList();
    // Start replicator stats
    topicStatsStream.startObject("replication");
    nsStats.replicatorCount += topicStats.remotePublishersStats.size();
    replicators.forEach((cluster, replicator) -> {
        // Update replicator cursor state
        ((PersistentReplicator) replicator).updateCursorState();
        // Update replicator stats
        ReplicatorStats rStat = replicator.getStats();
        // Add incoming msg rates
        PublisherStats pubStats = topicStats.remotePublishersStats.get(replicator.getRemoteCluster());
        if (pubStats != null) {
            rStat.msgRateIn = pubStats.msgRateIn;
            rStat.msgThroughputIn = pubStats.msgThroughputIn;
            rStat.inboundConnection = pubStats.address;
            rStat.inboundConnectedSince = pubStats.connectedSince;
        }
        topicStats.aggMsgRateOut += rStat.msgRateOut;
        topicStats.aggMsgThroughputOut += rStat.msgThroughputOut;
        // Populate replicator specific stats here
        topicStatsStream.startObject(cluster);
        topicStatsStream.writePair("connected", rStat.connected);
        topicStatsStream.writePair("msgRateExpired", rStat.msgRateExpired);
        topicStatsStream.writePair("msgRateIn", rStat.msgRateIn);
        topicStatsStream.writePair("msgRateOut", rStat.msgRateOut);
        topicStatsStream.writePair("msgThroughputIn", rStat.msgThroughputIn);
        topicStatsStream.writePair("msgThroughputOut", rStat.msgThroughputOut);
        topicStatsStream.writePair("replicationBacklog", rStat.replicationBacklog);
        topicStatsStream.writePair("replicationDelayInSeconds", rStat.replicationDelayInSeconds);
        topicStatsStream.writePair("inboundConnection", rStat.inboundConnection);
        topicStatsStream.writePair("inboundConnectedSince", rStat.inboundConnectedSince);
        topicStatsStream.writePair("outboundConnection", rStat.outboundConnection);
        topicStatsStream.writePair("outboundConnectedSince", rStat.outboundConnectedSince);
        topicStatsStream.endObject();
        nsStats.msgReplBacklog += rStat.replicationBacklog;
        if (replStats.isMetricsEnabled()) {
            String namespaceClusterKey = replStats.getKeyName(namespace, cluster);
            ReplicationMetrics replicationMetrics = replStats.get(namespaceClusterKey);
            boolean update = false;
            if (replicationMetrics == null) {
                replicationMetrics = ReplicationMetrics.get();
                update = true;
            }
            replicationMetrics.connected += rStat.connected ? 1 : 0;
            replicationMetrics.msgRateOut += rStat.msgRateOut;
            replicationMetrics.msgThroughputOut += rStat.msgThroughputOut;
            replicationMetrics.msgReplBacklog += rStat.replicationBacklog;
            if (update) {
                replStats.put(namespaceClusterKey, replicationMetrics);
            }
        }
    });
    // Close replication
    topicStatsStream.endObject();
    // Start subscription stats
    topicStatsStream.startObject("subscriptions");
    nsStats.subsCount += subscriptions.size();
    subscriptions.forEach((subscriptionName, subscription) -> {
        double subMsgRateOut = 0;
        double subMsgThroughputOut = 0;
        double subMsgRateRedeliver = 0;
        // Start subscription name & consumers
        try {
            topicStatsStream.startObject(subscriptionName);
            Object[] consumers = subscription.getConsumers().array();
            nsStats.consumerCount += consumers.length;
            bundleStats.consumerCount += consumers.length;
            topicStatsStream.startList("consumers");
            for (Object consumerObj : consumers) {
                Consumer consumer = (Consumer) consumerObj;
                consumer.updateRates();
                ConsumerStats consumerStats = consumer.getStats();
                subMsgRateOut += consumerStats.msgRateOut;
                subMsgThroughputOut += consumerStats.msgThroughputOut;
                subMsgRateRedeliver += consumerStats.msgRateRedeliver;
                // Populate consumer specific stats here
                topicStatsStream.startObject();
                topicStatsStream.writePair("address", consumerStats.address);
                topicStatsStream.writePair("consumerName", consumerStats.consumerName);
                topicStatsStream.writePair("availablePermits", consumerStats.availablePermits);
                topicStatsStream.writePair("connectedSince", consumerStats.connectedSince);
                topicStatsStream.writePair("msgRateOut", consumerStats.msgRateOut);
                topicStatsStream.writePair("msgThroughputOut", consumerStats.msgThroughputOut);
                topicStatsStream.writePair("msgRateRedeliver", consumerStats.msgRateRedeliver);
                if (SubType.Shared.equals(subscription.getType())) {
                    topicStatsStream.writePair("unackedMessages", consumerStats.unackedMessages);
                    topicStatsStream.writePair("blockedConsumerOnUnackedMsgs", consumerStats.blockedConsumerOnUnackedMsgs);
                }
                if (consumerStats.clientVersion != null) {
                    topicStatsStream.writePair("clientVersion", consumerStats.clientVersion);
                }
                topicStatsStream.endObject();
            }
            // Close Consumer stats
            topicStatsStream.endList();
            // Populate subscription specific stats here
            topicStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog());
            topicStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate());
            topicStatsStream.writePair("msgRateOut", subMsgRateOut);
            topicStatsStream.writePair("msgThroughputOut", subMsgThroughputOut);
            topicStatsStream.writePair("msgRateRedeliver", subMsgRateRedeliver);
            topicStatsStream.writePair("numberOfEntriesSinceFirstNotAckedMessage", subscription.getNumberOfEntriesSinceFirstNotAckedMessage());
            topicStatsStream.writePair("totalNonContiguousDeletedMessagesRange", subscription.getTotalNonContiguousDeletedMessagesRange());
            topicStatsStream.writePair("type", subscription.getTypeString());
            if (SubType.Shared.equals(subscription.getType())) {
                if (subscription.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) {
                    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) subscription.getDispatcher();
                    topicStatsStream.writePair("blockedSubscriptionOnUnackedMsgs", dispatcher.isBlockedDispatcherOnUnackedMsgs());
                    topicStatsStream.writePair("unackedMessages", dispatcher.getTotalUnackedMessages());
                }
            }
            // Close consumers
            topicStatsStream.endObject();
            topicStats.aggMsgRateOut += subMsgRateOut;
            topicStats.aggMsgThroughputOut += subMsgThroughputOut;
            nsStats.msgBacklog += subscription.getNumberOfEntriesInBacklog();
        } catch (Exception e) {
            log.error("Got exception when creating consumer stats for subscription {}: {}", subscriptionName, e.getMessage(), e);
        }
    });
    // Close subscription
    topicStatsStream.endObject();
    // Remaining dest stats.
    topicStats.averageMsgSize = topicStats.aggMsgRateIn == 0.0 ? 0.0 : (topicStats.aggMsgThroughputIn / topicStats.aggMsgRateIn);
    topicStatsStream.writePair("producerCount", producers.size());
    topicStatsStream.writePair("averageMsgSize", topicStats.averageMsgSize);
    topicStatsStream.writePair("msgRateIn", topicStats.aggMsgRateIn);
    topicStatsStream.writePair("msgRateOut", topicStats.aggMsgRateOut);
    topicStatsStream.writePair("msgThroughputIn", topicStats.aggMsgThroughputIn);
    topicStatsStream.writePair("msgThroughputOut", topicStats.aggMsgThroughputOut);
    topicStatsStream.writePair("storageSize", ledger.getEstimatedBacklogSize());
    topicStatsStream.writePair("pendingAddEntriesCount", ((ManagedLedgerImpl) ledger).getPendingAddEntriesCount());
    nsStats.msgRateIn += topicStats.aggMsgRateIn;
    nsStats.msgRateOut += topicStats.aggMsgRateOut;
    nsStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    nsStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    nsStats.storageSize += ledger.getEstimatedBacklogSize();
    bundleStats.msgRateIn += topicStats.aggMsgRateIn;
    bundleStats.msgRateOut += topicStats.aggMsgRateOut;
    bundleStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    bundleStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    bundleStats.cacheSize += ((ManagedLedgerImpl) ledger).getCacheSize();
    // Close topic object
    topicStatsStream.endObject();
}