use of com.yahoo.pulsar.common.policies.data.ConsumerStats in project pulsar by yahoo.
the class PersistentSubscription method getStats.
public PersistentSubscriptionStats getStats() {
PersistentSubscriptionStats subStats = new PersistentSubscriptionStats();
Dispatcher dispatcher = this.dispatcher;
if (dispatcher != null) {
dispatcher.getConsumers().forEach(consumer -> {
ConsumerStats consumerStats = consumer.getStats();
subStats.consumers.add(consumerStats);
subStats.msgRateOut += consumerStats.msgRateOut;
subStats.msgThroughputOut += consumerStats.msgThroughputOut;
subStats.msgRateRedeliver += consumerStats.msgRateRedeliver;
subStats.unackedMessages += consumerStats.unackedMessages;
});
}
subStats.msgBacklog = getNumberOfEntriesInBacklog();
subStats.msgRateExpired = expiryMonitor.getMessageExpiryRate();
subStats.type = getType();
return subStats;
}
use of com.yahoo.pulsar.common.policies.data.ConsumerStats in project pulsar by yahoo.
the class PersistentTopic method updateRates.
public void updateRates(NamespaceStats nsStats, NamespaceBundleStats bundleStats, StatsOutputStream destStatsStream, ClusterReplicationMetrics replStats, String namespace) {
TopicStats topicStats = threadLocalTopicStats.get();
topicStats.reset();
replicators.forEach((region, replicator) -> replicator.updateRates());
nsStats.producerCount += producers.size();
bundleStats.producerCount += producers.size();
destStatsStream.startObject(topic);
producers.forEach(producer -> {
producer.updateRates();
PublisherStats publisherStats = producer.getStats();
topicStats.aggMsgRateIn += publisherStats.msgRateIn;
topicStats.aggMsgThroughputIn += publisherStats.msgThroughputIn;
if (producer.isRemote()) {
topicStats.remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
}
});
// Creating publishers object for backward compatibility
destStatsStream.startList("publishers");
destStatsStream.endList();
// Start replicator stats
destStatsStream.startObject("replication");
nsStats.replicatorCount += topicStats.remotePublishersStats.size();
replicators.forEach((cluster, replicator) -> {
// Update replicator cursor state
replicator.updateCursorState();
// Update replicator stats
ReplicatorStats rStat = replicator.getStats();
// Add incoming msg rates
PublisherStats pubStats = topicStats.remotePublishersStats.get(replicator.getRemoteCluster());
if (pubStats != null) {
rStat.msgRateIn = pubStats.msgRateIn;
rStat.msgThroughputIn = pubStats.msgThroughputIn;
rStat.inboundConnection = pubStats.address;
rStat.inboundConnectedSince = pubStats.connectedSince;
}
topicStats.aggMsgRateOut += rStat.msgRateOut;
topicStats.aggMsgThroughputOut += rStat.msgThroughputOut;
// Populate replicator specific stats here
destStatsStream.startObject(cluster);
destStatsStream.writePair("connected", rStat.connected);
destStatsStream.writePair("msgRateExpired", rStat.msgRateExpired);
destStatsStream.writePair("msgRateIn", rStat.msgRateIn);
destStatsStream.writePair("msgRateOut", rStat.msgRateOut);
destStatsStream.writePair("msgThroughputIn", rStat.msgThroughputIn);
destStatsStream.writePair("msgThroughputOut", rStat.msgThroughputOut);
destStatsStream.writePair("replicationBacklog", rStat.replicationBacklog);
destStatsStream.writePair("replicationDelayInSeconds", rStat.replicationDelayInSeconds);
destStatsStream.writePair("inboundConnection", rStat.inboundConnection);
destStatsStream.writePair("inboundConnectedSince", rStat.inboundConnectedSince);
destStatsStream.writePair("outboundConnection", rStat.outboundConnection);
destStatsStream.writePair("outboundConnectedSince", rStat.outboundConnectedSince);
destStatsStream.endObject();
nsStats.msgReplBacklog += rStat.replicationBacklog;
if (replStats.isMetricsEnabled()) {
String namespaceClusterKey = replStats.getKeyName(namespace, cluster);
ReplicationMetrics replicationMetrics = replStats.get(namespaceClusterKey);
boolean update = false;
if (replicationMetrics == null) {
replicationMetrics = ReplicationMetrics.get();
update = true;
}
replicationMetrics.connected += rStat.connected ? 1 : 0;
replicationMetrics.msgRateOut += rStat.msgRateOut;
replicationMetrics.msgThroughputOut += rStat.msgThroughputOut;
replicationMetrics.msgReplBacklog += rStat.replicationBacklog;
if (update) {
replStats.put(namespaceClusterKey, replicationMetrics);
}
}
});
// Close replication
destStatsStream.endObject();
// Start subscription stats
destStatsStream.startObject("subscriptions");
nsStats.subsCount += subscriptions.size();
subscriptions.forEach((subscriptionName, subscription) -> {
double subMsgRateOut = 0;
double subMsgThroughputOut = 0;
double subMsgRateRedeliver = 0;
long subUnackedMessages = 0;
// Start subscription name & consumers
try {
destStatsStream.startObject(subscriptionName);
Object[] consumers = subscription.getConsumers().array();
nsStats.consumerCount += consumers.length;
bundleStats.consumerCount += consumers.length;
destStatsStream.startList("consumers");
for (Object consumerObj : consumers) {
Consumer consumer = (Consumer) consumerObj;
consumer.updateRates();
ConsumerStats consumerStats = consumer.getStats();
subMsgRateOut += consumerStats.msgRateOut;
subMsgThroughputOut += consumerStats.msgThroughputOut;
subMsgRateRedeliver += consumerStats.msgRateRedeliver;
subUnackedMessages += consumerStats.unackedMessages;
// Populate consumer specific stats here
destStatsStream.startObject();
destStatsStream.writePair("address", consumerStats.address);
destStatsStream.writePair("consumerName", consumerStats.consumerName);
destStatsStream.writePair("availablePermits", consumerStats.availablePermits);
destStatsStream.writePair("unackedMessages", consumerStats.unackedMessages);
destStatsStream.writePair("blockedConsumerOnUnackedMsgs", consumerStats.blockedConsumerOnUnackedMsgs);
destStatsStream.writePair("connectedSince", consumerStats.connectedSince);
destStatsStream.writePair("msgRateOut", consumerStats.msgRateOut);
destStatsStream.writePair("msgThroughputOut", consumerStats.msgThroughputOut);
destStatsStream.writePair("msgRateRedeliver", consumerStats.msgRateRedeliver);
destStatsStream.endObject();
}
// Close Consumer stats
destStatsStream.endList();
// Populate subscription specific stats here
destStatsStream.writePair("msgBacklog", subscription.getNumberOfEntriesInBacklog());
destStatsStream.writePair("msgRateExpired", subscription.getExpiredMessageRate());
destStatsStream.writePair("msgRateOut", subMsgRateOut);
destStatsStream.writePair("msgThroughputOut", subMsgThroughputOut);
destStatsStream.writePair("msgRateRedeliver", subMsgRateRedeliver);
destStatsStream.writePair("unackedMessages", subUnackedMessages);
destStatsStream.writePair("type", subscription.getTypeString());
// Close consumers
destStatsStream.endObject();
topicStats.aggMsgRateOut += subMsgRateOut;
topicStats.aggMsgThroughputOut += subMsgThroughputOut;
nsStats.msgBacklog += subscription.getNumberOfEntriesInBacklog();
} catch (Exception e) {
log.error("Got exception when creating consumer stats for subscription {}: {}", subscriptionName, e.getMessage(), e);
}
});
// Close subscription
destStatsStream.endObject();
// Remaining dest stats.
topicStats.averageMsgSize = topicStats.aggMsgRateIn == 0.0 ? 0.0 : (topicStats.aggMsgThroughputIn / topicStats.aggMsgRateIn);
destStatsStream.writePair("producerCount", producers.size());
destStatsStream.writePair("averageMsgSize", topicStats.averageMsgSize);
destStatsStream.writePair("msgRateIn", topicStats.aggMsgRateIn);
destStatsStream.writePair("msgRateOut", topicStats.aggMsgRateOut);
destStatsStream.writePair("msgThroughputIn", topicStats.aggMsgThroughputIn);
destStatsStream.writePair("msgThroughputOut", topicStats.aggMsgThroughputOut);
destStatsStream.writePair("storageSize", ledger.getEstimatedBacklogSize());
destStatsStream.writePair("pendingAddEntriesCount", ((ManagedLedgerImpl) ledger).getPendingAddEntriesCount());
nsStats.msgRateIn += topicStats.aggMsgRateIn;
nsStats.msgRateOut += topicStats.aggMsgRateOut;
nsStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
nsStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
nsStats.storageSize += ledger.getEstimatedBacklogSize();
bundleStats.msgRateIn += topicStats.aggMsgRateIn;
bundleStats.msgRateOut += topicStats.aggMsgRateOut;
bundleStats.msgThroughputIn += topicStats.aggMsgThroughputIn;
bundleStats.msgThroughputOut += topicStats.aggMsgThroughputOut;
bundleStats.cacheSize += ((ManagedLedgerImpl) ledger).getCacheSize();
// Close topic object
destStatsStream.endObject();
}
use of com.yahoo.pulsar.common.policies.data.ConsumerStats in project pulsar by yahoo.
the class ServerCnx method createConsumerStatsResponse.
CommandConsumerStatsResponse.Builder createConsumerStatsResponse(Consumer consumer, PersistentSubscription subscription, long requestId) {
CommandConsumerStatsResponse.Builder commandConsumerStatsResponseBuilder = CommandConsumerStatsResponse.newBuilder();
ConsumerStats consumerStats = consumer.getStats();
commandConsumerStatsResponseBuilder.setRequestId(requestId);
commandConsumerStatsResponseBuilder.setMsgRateOut(consumerStats.msgRateOut);
commandConsumerStatsResponseBuilder.setMsgThroughputOut(consumerStats.msgThroughputOut);
commandConsumerStatsResponseBuilder.setMsgRateRedeliver(consumerStats.msgRateRedeliver);
commandConsumerStatsResponseBuilder.setConsumerName(consumerStats.consumerName);
commandConsumerStatsResponseBuilder.setAvailablePermits(consumerStats.availablePermits);
commandConsumerStatsResponseBuilder.setUnackedMessages(consumerStats.unackedMessages);
commandConsumerStatsResponseBuilder.setBlockedConsumerOnUnackedMsgs(consumerStats.blockedConsumerOnUnackedMsgs);
commandConsumerStatsResponseBuilder.setAddress(consumerStats.address);
commandConsumerStatsResponseBuilder.setConnectedSince(consumerStats.connectedSince);
commandConsumerStatsResponseBuilder.setType(subscription.getTypeString());
commandConsumerStatsResponseBuilder.setMsgRateExpired(subscription.getExpiredMessageRate());
commandConsumerStatsResponseBuilder.setMsgBacklog(subscription.getNumberOfEntriesInBacklog());
return commandConsumerStatsResponseBuilder;
}
Aggregations