Examples with ReplicatorStats com.yahoo.pulsar.common.policies.data.ReplicatorStats used on opensource projects

Example 1 with ReplicatorStats

use of com.yahoo.pulsar.common.policies.data.ReplicatorStats in project pulsar by yahoo.

the class PersistentTopic method getStats.

public PersistentTopicStats getStats() {
    PersistentTopicStats stats = new PersistentTopicStats();
    ObjectObjectHashMap<String, PublisherStats> remotePublishersStats = new ObjectObjectHashMap<String, PublisherStats>();
    producers.forEach(producer -> {
        PublisherStats publisherStats = producer.getStats();
        stats.msgRateIn += publisherStats.msgRateIn;
        stats.msgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        } else {
            stats.publishers.add(publisherStats);
        }
    });
    stats.averageMsgSize = stats.msgRateIn == 0.0 ? 0.0 : (stats.msgThroughputIn / stats.msgRateIn);
    subscriptions.forEach((name, subscription) -> {
        PersistentSubscriptionStats subStats = subscription.getStats();
        stats.msgRateOut += subStats.msgRateOut;
        stats.msgThroughputOut += subStats.msgThroughputOut;
        stats.subscriptions.put(name, subStats);
    });
    replicators.forEach((cluster, replicator) -> {
        ReplicatorStats replicatorStats = replicator.getStats();
        // Add incoming msg rates
        PublisherStats pubStats = remotePublishersStats.get(replicator.getRemoteCluster());
        if (pubStats != null) {
            replicatorStats.msgRateIn = pubStats.msgRateIn;
            replicatorStats.msgThroughputIn = pubStats.msgThroughputIn;
            replicatorStats.inboundConnection = pubStats.address;
            replicatorStats.inboundConnectedSince = pubStats.connectedSince;
        }
        stats.msgRateOut += replicatorStats.msgRateOut;
        stats.msgThroughputOut += replicatorStats.msgThroughputOut;
        stats.replication.put(replicator.getRemoteCluster(), replicatorStats);
    });
    stats.storageSize = ledger.getEstimatedBacklogSize();
    return stats;
}

Example 2 with ReplicatorStats

use of com.yahoo.pulsar.common.policies.data.ReplicatorStats in project pulsar by yahoo.

the class PersistentTopic method updateRates.

public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats, StatsOutputStream destStatsStream, ClusterReplicationMetrics replStats, String namespace) {
    TopicStats topicStats = threadLocalTopicStats.get();
    topicStats.reset();
    replicators.forEach((region, replicator) -> replicator.updateRates());
    nsStats.producerCount += producers.size();
    bundleStats.producerCount += producers.size();
    destStatsStream.startObject(topic);
    producers.forEach(producer -> {
        producer.updateRates();
        PublisherStats publisherStats = producer.getStats();
        topicStats.aggMsgRateIn += publisherStats.msgRateIn;
        topicStats.aggMsgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            topicStats.remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        }
    });
    // Creating publishers object for backward compatibility
    destStatsStream.startList("publishers");
    destStatsStream.endList();
    // Start replicator stats
    destStatsStream.startObject("replication");
    nsStats.replicatorCount += topicStats.remotePublishersStats.size();
    replicators.forEach((cluster, replicator) -> {
        // Update replicator cursor state
        replicator.updateCursorState();
        // Update replicator stats
        ReplicatorStats rStat = replicator.getStats();
        // Add incoming msg rates
        PublisherStats pubStats = topicStats.remotePublishersStats.get(replicator.getRemoteCluster());
        if (pubStats != null) {
            rStat.msgRateIn = pubStats.msgRateIn;
            rStat.msgThroughputIn = pubStats.msgThroughputIn;
            rStat.inboundConnection = pubStats.address;
            rStat.inboundConnectedSince = pubStats.connectedSince;
        }
        topicStats.aggMsgRateOut += rStat.msgRateOut;
        topicStats.aggMsgThroughputOut += rStat.msgThroughputOut;
        // Populate replicator specific stats here
        destStatsStream.startObject(cluster);
        destStatsStream.writePair("connected", rStat.connected);
        destStatsStream.writePair("msgRateExpired", rStat.msgRateExpired);
        destStatsStream.writePair("msgRateIn", rStat.msgRateIn);
        destStatsStream.writePair("msgRateOut", rStat.msgRateOut);
        destStatsStream.writePair("msgThroughputIn", rStat.msgThroughputIn);
        destStatsStream.writePair("msgThroughputOut", rStat.msgThroughputOut);
        destStatsStream.writePair("replicationBacklog", rStat.replicationBacklog);
        destStatsStream.writePair("replicationDelayInSeconds", rStat.replicationDelayInSeconds);
        destStatsStream.writePair("inboundConnection", rStat.inboundConnection);
        destStatsStream.writePair("inboundConnectedSince", rStat.inboundConnectedSince);
        destStatsStream.writePair("outboundConnection", rStat.outboundConnection);
        destStatsStream.writePair("outboundConnectedSince", rStat.outboundConnectedSince);
        destStatsStream.endObject();
        nsStats.msgReplBacklog += rStat.replicationBacklog;
        if (replStats.isMetricsEnabled()) {
            String namespaceClusterKey = replStats.getKeyName(namespace, cluster);
            ReplicationMetrics replicationMetrics = replStats.get(namespaceClusterKey);
            boolean update = false;
            if (replicationMetrics == null) {
                replicationMetrics = ReplicationMetrics.get();
                update = true;
            }
            replicationMetrics.connected += rStat.connected ? 1 : 0;
            replicationMetrics.msgRateOut += rStat.msgRateOut;
            replicationMetrics.msgThroughputOut += rStat.msgThroughputOut;
            replicationMetrics.msgReplBacklog += rStat.replicationBacklog;
            if (update) {
                replStats.put(namespaceClusterKey, replicationMetrics);
            }
        }
    });
    // Close replication
    destStatsStream.endObject();
    // Start subscription stats
    destStatsStream.startObject("subscriptions");
    nsStats.subsCount += subscriptions.size();
    subscriptions.forEach((subscriptionName, subscription) -> {
        double subMsgRateOut = 0;
        double subMsgThroughputOut = 0;
        double subMsgRateRedeliver = 0;
        long subUnackedMessages = 0;
        // Start subscription name & consumers
        try {
            destStatsStream.startObject(subscriptionName);
            Object[] consumers = subscription.getConsumers().array();
            nsStats.consumerCount += consumers.length;
            bundleStats.consumerCount += consumers.length;
            destStatsStream.startList("consumers");
            for (Object consumerObj : consumers) {
                Consumer consumer = (Consumer) consumerObj;
                consumer.updateRates();
                ConsumerStats consumerStats = consumer.getStats();
                subMsgRateOut += consumerStats.msgRateOut;
                subMsgThroughputOut += consumerStats.msgThroughputOut;
                subMsgRateRedeliver += consumerStats.msgRateRedeliver;
                subUnackedMessages += consumerStats.unackedMessages;
                // Populate consumer specific stats here
                destStatsStream.startObject();
                destStatsStream.writePair("address", consumerStats.address);
                destStatsStream.writePair("consumerName", consumerStats.consumerName);
                destStatsStream.writePair("availablePermits", consumerStats.availablePermits);
                destStatsStream.writePair("unackedMessages", consumerStats.unackedMessages);
                destStatsStream.writePair("blockedConsumerOnUnackedMsgs", consumerStats.blockedConsumerOnUnackedMsgs);
                destStatsStream.writePair("connectedSince", consumerStats.connectedSince);
                destStatsStream.writePair("msgRateOut", consumerStats.msgRateOut);
                destStatsStream.writePair("msgThroughputOut", consumerStats.msgThroughputOut);
                destStatsStream.writePair("msgRateRedeliver", consumerStats.msgRateRedeliver);
                destStatsStream.endObject();
            }
            // Close Consumer stats
            destStatsStream.endList();
            // Populate subscription specific stats here
            destStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog());
            destStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate());
            destStatsStream.writePair("msgRateOut", subMsgRateOut);
            destStatsStream.writePair("msgThroughputOut", subMsgThroughputOut);
            destStatsStream.writePair("msgRateRedeliver", subMsgRateRedeliver);
            destStatsStream.writePair("unackedMessages", subUnackedMessages);
            destStatsStream.writePair("type", subscription.getTypeString());
            // Close consumers
            destStatsStream.endObject();
            topicStats.aggMsgRateOut += subMsgRateOut;
            topicStats.aggMsgThroughputOut += subMsgThroughputOut;
            nsStats.msgBacklog += subscription.getNumberOfEntriesInBacklog();
        } catch (Exception e) {
            log.error("Got exception when creating consumer stats for subscription {}: {}", subscriptionName, e.getMessage(), e);
        }
    });
    // Close subscription
    destStatsStream.endObject();
    // Remaining dest stats.
    topicStats.averageMsgSize = topicStats.aggMsgRateIn == 0.0 ? 0.0 : (topicStats.aggMsgThroughputIn / topicStats.aggMsgRateIn);
    destStatsStream.writePair("producerCount", producers.size());
    destStatsStream.writePair("averageMsgSize", topicStats.averageMsgSize);
    destStatsStream.writePair("msgRateIn", topicStats.aggMsgRateIn);
    destStatsStream.writePair("msgRateOut", topicStats.aggMsgRateOut);
    destStatsStream.writePair("msgThroughputIn", topicStats.aggMsgThroughputIn);
    destStatsStream.writePair("msgThroughputOut", topicStats.aggMsgThroughputOut);
    destStatsStream.writePair("storageSize", ledger.getEstimatedBacklogSize());
    destStatsStream.writePair("pendingAddEntriesCount", ((ManagedLedgerImpl) ledger).getPendingAddEntriesCount());
    nsStats.msgRateIn += topicStats.aggMsgRateIn;
    nsStats.msgRateOut += topicStats.aggMsgRateOut;
    nsStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    nsStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    nsStats.storageSize += ledger.getEstimatedBacklogSize();
    bundleStats.msgRateIn += topicStats.aggMsgRateIn;
    bundleStats.msgRateOut += topicStats.aggMsgRateOut;
    bundleStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
    bundleStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
    bundleStats.cacheSize += ((ManagedLedgerImpl) ledger).getCacheSize();
    // Close topic object
    destStatsStream.endObject();
}

Example 3 with ReplicatorStats

use of com.yahoo.pulsar.common.policies.data.ReplicatorStats in project pulsar by yahoo.

the class ReplicatorTest method testReplicatorClearBacklog.

@Test
public void testReplicatorClearBacklog() throws Exception {
    // This test is to verify that reset cursor fails on global topic
    SortedSet<String> testDests = new TreeSet<String>();
    final DestinationName dest = DestinationName.get("persistent://pulsar/global/ns/clearBacklogTopic");
    testDests.add(dest.toString());
    MessageProducer producer1 = new MessageProducer(url1, dest);
    MessageConsumer consumer1 = new MessageConsumer(url3, dest);
    // Produce from cluster1 and consume from the rest
    producer1.produce(2);
    producer1.close();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopicReference(dest.toString());
    PersistentReplicator replicator = spy(topic.getReplicators().get(topic.getReplicators().keys().get(0)));
    replicator.readEntriesFailed(new ManagedLedgerException.InvalidCursorPositionException("failed"), null);
    replicator.clearBacklog().get();
    Thread.sleep(100);
    // for code-coverage
    replicator.updateRates();
    // for code-coverage
    replicator.expireMessages(1);
    ReplicatorStats status = replicator.getStats();
    assertTrue(status.replicationBacklog == 0);
    consumer1.close();
}