use of org.apache.pulsar.common.policies.data.stats.TopicStatsImpl in project pulsar by apache.
the class PersistentTopicsBase method internalGetPartitionedStats.
protected void internalGetPartitionedStats(AsyncResponse asyncResponse, boolean authoritative, boolean perPartition, boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) {
CompletableFuture<Void> future;
if (topicName.isGlobal()) {
future = validateGlobalNamespaceOwnershipAsync(namespaceName);
} else {
future = CompletableFuture.completedFuture(null);
}
future.thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)).thenAccept(partitionMetadata -> {
if (partitionMetadata.partitions == 0) {
asyncResponse.resume(new RestException(Status.NOT_FOUND, "Partitioned Topic not found"));
return;
}
PartitionedTopicStatsImpl stats = new PartitionedTopicStatsImpl(partitionMetadata);
List<CompletableFuture<TopicStats>> topicStatsFutureList = Lists.newArrayList();
for (int i = 0; i < partitionMetadata.partitions; i++) {
try {
topicStatsFutureList.add(pulsar().getAdminClient().topics().getStatsAsync((topicName.getPartition(i).toString()), getPreciseBacklog, subscriptionBacklogSize, getEarliestTimeInBacklog));
} catch (PulsarServerException e) {
asyncResponse.resume(new RestException(e));
return;
}
}
FutureUtil.waitForAll(topicStatsFutureList).handle((result, exception) -> {
CompletableFuture<TopicStats> statFuture = null;
for (int i = 0; i < topicStatsFutureList.size(); i++) {
statFuture = topicStatsFutureList.get(i);
if (statFuture.isDone() && !statFuture.isCompletedExceptionally()) {
try {
stats.add(statFuture.get());
if (perPartition) {
stats.getPartitions().put(topicName.getPartition(i).toString(), (TopicStatsImpl) statFuture.get());
}
} catch (Exception e) {
asyncResponse.resume(new RestException(e));
return null;
}
}
}
if (perPartition && stats.partitions.isEmpty()) {
try {
boolean pathExists = namespaceResources().getPartitionedTopicResources().partitionedTopicExists(topicName);
if (pathExists) {
stats.partitions.put(topicName.toString(), new TopicStatsImpl());
} else {
asyncResponse.resume(new RestException(Status.NOT_FOUND, "Internal topics have not been generated yet"));
return null;
}
} catch (Exception e) {
asyncResponse.resume(new RestException(e));
return null;
}
}
asyncResponse.resume(stats);
return null;
});
}).exceptionally(ex -> {
// If the exception is not redirect exception we need to log it.
if (!isRedirectException(ex)) {
log.error("[{}] Failed to get partitioned internal stats for {}", clientAppId(), topicName, ex);
}
resumeAsyncResponseExceptionally(asyncResponse, ex);
return null;
});
}
use of org.apache.pulsar.common.policies.data.stats.TopicStatsImpl in project pulsar by apache.
the class RGUsageMTAggrWaitForAllMsgsTest method verifyRGProdConsStats.
// Verify the app stats with what we see from the broker-service, and the resource-group (which in turn internally
// derives stats from the broker service)
private void verifyRGProdConsStats(String[] topicStrings, int sentNumBytes, int sentNumMsgs, int recvdNumBytes, int recvdNumMsgs, int scaleFactor, boolean checkProduce, boolean checkConsume) throws Exception {
BrokerService bs = pulsar.getBrokerService();
Map<String, TopicStatsImpl> topicStatsMap = bs.getTopicStats();
log.debug("verifyProdConsStats: topicStatsMap has {} entries", topicStatsMap.size());
// Pulsar runtime adds some additional bytes in the exchanges: a 45-byte per-message
// metadata of some kind, plus more as the number of messages increases.
// Hence the ">=" assertion with ExpectedNumBytesSent/Received in the following checks.
final int ExpectedNumBytesSent = sentNumBytes + PER_MESSAGE_METADATA_OHEAD * sentNumMsgs;
final int ExpectedNumBytesReceived = recvdNumBytes + PER_MESSAGE_METADATA_OHEAD * recvdNumMsgs;
long totalOutMessages = 0, totalOutBytes = 0;
long totalInMessages = 0, totalInBytes = 0;
BytesAndMessagesCount totalTenantRGProdCounts = new BytesAndMessagesCount();
BytesAndMessagesCount totalTenantRGConsCounts = new BytesAndMessagesCount();
BytesAndMessagesCount totalNsRGProdCounts = new BytesAndMessagesCount();
BytesAndMessagesCount totalNsRGConsCounts = new BytesAndMessagesCount();
BytesAndMessagesCount prodCounts, consCounts;
// Since the following walk is on topics, keep track of the RGs for which we have already gathered stats,
// so that we do not double-accumulate stats if multiple topics refer to the same RG.
HashSet<String> RGsWithPublishStatsGathered = new HashSet<>();
HashSet<String> RGsWithDispatchStatsGathered = new HashSet<>();
// Hack to ensure aggregator calculation without waiting for a period of aggregation.
// [aggregateResourceGroupLocalUsages() is idempotent when there's no fresh traffic flowing.]
this.rgservice.aggregateResourceGroupLocalUsages();
for (Map.Entry<String, TopicStatsImpl> entry : topicStatsMap.entrySet()) {
String mapTopicName = entry.getKey();
if (Arrays.asList(topicStrings).contains(mapTopicName)) {
TopicStats stats = entry.getValue();
totalInMessages += stats.getMsgInCounter();
totalInBytes += stats.getBytesInCounter();
totalOutMessages += stats.getMsgOutCounter();
totalOutBytes += stats.getBytesOutCounter();
// we should see some produced mesgs on every topic.
if (totalInMessages == 0) {
log.warn("verifyProdConsStats: found no produced mesgs (msgInCounter) on topic {}", mapTopicName);
}
if (sentNumMsgs > 0 || recvdNumMsgs > 0) {
TopicName topic = TopicName.get(mapTopicName);
final String tenantRGName = TopicToTenantRGName(topic);
if (!RGsWithPublishStatsGathered.contains(tenantRGName)) {
prodCounts = this.rgservice.getRGUsage(tenantRGName, ResourceGroupMonitoringClass.Publish, getCumulativeUsageStats);
totalTenantRGProdCounts = ResourceGroup.accumulateBMCount(totalTenantRGProdCounts, prodCounts);
RGsWithPublishStatsGathered.add(tenantRGName);
}
if (!RGsWithDispatchStatsGathered.contains(tenantRGName)) {
consCounts = this.rgservice.getRGUsage(tenantRGName, ResourceGroupMonitoringClass.Dispatch, getCumulativeUsageStats);
totalTenantRGConsCounts = ResourceGroup.accumulateBMCount(totalTenantRGConsCounts, consCounts);
RGsWithDispatchStatsGathered.add(tenantRGName);
}
final String nsRGName = TopicToNamespaceRGName(topic);
// We will do the same here, to get the expected stats.
if (tenantRGName.compareTo(nsRGName) != 0) {
if (!RGsWithPublishStatsGathered.contains(nsRGName)) {
prodCounts = this.rgservice.getRGUsage(nsRGName, ResourceGroupMonitoringClass.Publish, getCumulativeUsageStats);
totalNsRGProdCounts = ResourceGroup.accumulateBMCount(totalNsRGProdCounts, prodCounts);
RGsWithPublishStatsGathered.add(nsRGName);
}
if (!RGsWithDispatchStatsGathered.contains(nsRGName)) {
consCounts = this.rgservice.getRGUsage(nsRGName, ResourceGroupMonitoringClass.Dispatch, getCumulativeUsageStats);
totalNsRGConsCounts = ResourceGroup.accumulateBMCount(totalNsRGConsCounts, consCounts);
RGsWithDispatchStatsGathered.add(nsRGName);
}
}
}
}
}
// Check that the accumulated totals tally up.
if (checkConsume && checkProduce) {
Assert.assertEquals(totalOutMessages, totalInMessages);
Assert.assertEquals(totalOutBytes, totalInBytes);
}
if (checkProduce) {
Assert.assertEquals(totalInMessages, sentNumMsgs);
Assert.assertTrue(totalInBytes >= ExpectedNumBytesSent);
}
if (checkConsume) {
Assert.assertEquals(totalOutMessages, recvdNumMsgs);
Assert.assertTrue(totalOutBytes >= ExpectedNumBytesReceived);
}
if (checkProduce) {
prodCounts = ResourceGroup.accumulateBMCount(totalTenantRGProdCounts, totalNsRGProdCounts);
Assert.assertEquals(prodCounts.messages, sentNumMsgs * scaleFactor);
Assert.assertTrue(prodCounts.bytes >= ExpectedNumBytesSent);
}
if (checkConsume) {
consCounts = ResourceGroup.accumulateBMCount(totalTenantRGConsCounts, totalNsRGConsCounts);
Assert.assertEquals(consCounts.messages, recvdNumMsgs * scaleFactor);
Assert.assertTrue(consCounts.bytes >= ExpectedNumBytesReceived);
}
}
use of org.apache.pulsar.common.policies.data.stats.TopicStatsImpl in project pulsar by apache.
the class PersistentTopicStatsTest method testPersistentTopicStatsAggregationPartialProducerIsNotSupported.
@Test
public void testPersistentTopicStatsAggregationPartialProducerIsNotSupported() {
TopicStatsImpl topicStats1 = new TopicStatsImpl();
topicStats1.msgRateIn = 1;
topicStats1.msgThroughputIn = 1;
topicStats1.msgRateOut = 1;
topicStats1.msgThroughputOut = 1;
topicStats1.averageMsgSize = 1;
topicStats1.storageSize = 1;
final PublisherStatsImpl publisherStats1 = new PublisherStatsImpl();
publisherStats1.setSupportsPartialProducer(false);
publisherStats1.setProducerName("name1");
topicStats1.addPublisher(publisherStats1);
topicStats1.subscriptions.put("test_ns", new SubscriptionStatsImpl());
topicStats1.replication.put("test_ns", new ReplicatorStatsImpl());
TopicStatsImpl topicStats2 = new TopicStatsImpl();
topicStats2.msgRateIn = 1;
topicStats2.msgThroughputIn = 2;
topicStats2.msgRateOut = 3;
topicStats2.msgThroughputOut = 4;
topicStats2.averageMsgSize = 5;
topicStats2.storageSize = 6;
final PublisherStatsImpl publisherStats2 = new PublisherStatsImpl();
publisherStats2.setSupportsPartialProducer(false);
publisherStats2.setProducerName("name1");
topicStats2.addPublisher(publisherStats2);
topicStats2.subscriptions.put("test_ns", new SubscriptionStatsImpl());
topicStats2.replication.put("test_ns", new ReplicatorStatsImpl());
TopicStatsImpl target = new TopicStatsImpl();
target.add(topicStats1);
target.add(topicStats2);
assertEquals(target.msgRateIn, 2.0);
assertEquals(target.msgThroughputIn, 3.0);
assertEquals(target.msgRateOut, 4.0);
assertEquals(target.msgThroughputOut, 5.0);
assertEquals(target.averageMsgSize, 3.0);
assertEquals(target.storageSize, 7);
assertEquals(target.getPublishers().size(), 1);
assertEquals(target.subscriptions.size(), 1);
assertEquals(target.replication.size(), 1);
}
use of org.apache.pulsar.common.policies.data.stats.TopicStatsImpl in project pulsar by apache.
the class PersistentTopicStatsTest method testPersistentTopicStatsAggregationByProducerName.
@Test
public void testPersistentTopicStatsAggregationByProducerName() {
TopicStatsImpl topicStats1 = new TopicStatsImpl();
topicStats1.msgRateIn = 1;
topicStats1.msgThroughputIn = 1;
topicStats1.msgRateOut = 1;
topicStats1.msgThroughputOut = 1;
topicStats1.averageMsgSize = 1;
topicStats1.storageSize = 1;
final PublisherStatsImpl publisherStats1 = new PublisherStatsImpl();
publisherStats1.setSupportsPartialProducer(true);
publisherStats1.msgRateIn = 1;
publisherStats1.setProducerName("name1");
topicStats1.addPublisher(publisherStats1);
topicStats1.subscriptions.put("test_ns", new SubscriptionStatsImpl());
topicStats1.replication.put("test_ns", new ReplicatorStatsImpl());
TopicStatsImpl topicStats2 = new TopicStatsImpl();
topicStats2.msgRateIn = 1;
topicStats2.msgThroughputIn = 2;
topicStats2.msgRateOut = 3;
topicStats2.msgThroughputOut = 4;
topicStats2.averageMsgSize = 5;
topicStats2.storageSize = 6;
final PublisherStatsImpl publisherStats2 = new PublisherStatsImpl();
publisherStats2.setSupportsPartialProducer(true);
publisherStats2.msgRateIn = 1;
publisherStats2.setProducerName("name1");
topicStats2.addPublisher(publisherStats2);
topicStats2.subscriptions.put("test_ns", new SubscriptionStatsImpl());
topicStats2.replication.put("test_ns", new ReplicatorStatsImpl());
TopicStatsImpl topicStats3 = new TopicStatsImpl();
topicStats3.msgRateIn = 0;
topicStats3.msgThroughputIn = 0;
topicStats3.msgRateOut = 0;
topicStats3.msgThroughputOut = 0;
topicStats3.averageMsgSize = 0;
topicStats3.storageSize = 0;
final PublisherStatsImpl publisherStats3 = new PublisherStatsImpl();
publisherStats3.setSupportsPartialProducer(true);
publisherStats3.msgRateIn = 1;
publisherStats3.setProducerName("name2");
topicStats3.addPublisher(publisherStats3);
topicStats3.subscriptions.put("test_ns", new SubscriptionStatsImpl());
topicStats3.replication.put("test_ns", new ReplicatorStatsImpl());
TopicStatsImpl target = new TopicStatsImpl();
target.add(topicStats1);
target.add(topicStats2);
target.add(topicStats3);
assertEquals(target.msgRateIn, 2.0);
assertEquals(target.msgThroughputIn, 3.0);
assertEquals(target.msgRateOut, 4.0);
assertEquals(target.msgThroughputOut, 5.0);
assertEquals(target.averageMsgSize, 2.0);
assertEquals(target.storageSize, 7);
assertEquals(target.getPublishers().size(), 2);
final Map<String, Double> expectedPublishersMap = Maps.newHashMap();
expectedPublishersMap.put("name1", 2.0);
expectedPublishersMap.put("name2", 1.0);
assertEquals(target.getPublishers().stream().collect(Collectors.toMap(PublisherStats::getProducerName, e -> ((PublisherStatsImpl) e).msgRateIn)), expectedPublishersMap);
assertEquals(target.subscriptions.size(), 1);
assertEquals(target.replication.size(), 1);
}
use of org.apache.pulsar.common.policies.data.stats.TopicStatsImpl in project pulsar by apache.
the class ResourceGroupUsageAggregationTest method verifyStats.
// Verify the app stats with what we see from the broker-service, and the resource-group (which in turn internally
// derives stats from the broker service)
// There appears to be a 45-byte message header which is accounted in the stats, additionally to what the
// application-level sends/receives. Hence, the byte counts are a ">=" check, instead of an equality check.
private void verifyStats(String topicString, String rgName, int sentNumBytes, int sentNumMsgs, int recvdNumBytes, int recvdNumMsgs, boolean checkProduce, boolean checkConsume) throws InterruptedException, PulsarAdminException {
BrokerService bs = pulsar.getBrokerService();
Map<String, TopicStatsImpl> topicStatsMap = bs.getTopicStats();
for (Map.Entry<String, TopicStatsImpl> entry : topicStatsMap.entrySet()) {
String mapTopicName = entry.getKey();
if (mapTopicName.equals(topicString)) {
TopicStatsImpl stats = entry.getValue();
if (checkProduce) {
Assert.assertTrue(stats.bytesInCounter >= sentNumBytes);
Assert.assertEquals(sentNumMsgs, stats.msgInCounter);
}
if (checkConsume) {
Assert.assertTrue(stats.bytesOutCounter >= recvdNumBytes);
Assert.assertEquals(recvdNumMsgs, stats.msgOutCounter);
}
if (sentNumMsgs > 0 || recvdNumMsgs > 0) {
// hack to ensure aggregator calculation without waiting
rgs.aggregateResourceGroupLocalUsages();
BytesAndMessagesCount prodCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Publish, ResourceGroupUsageStatsType.Cumulative);
BytesAndMessagesCount consCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Dispatch, ResourceGroupUsageStatsType.Cumulative);
// Re-do the getRGUsage.
// The counts should be equal, since there wasn't any intervening traffic on TEST_PRODUCE_CONSUME_TOPIC.
BytesAndMessagesCount prodCounts1 = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Publish, ResourceGroupUsageStatsType.Cumulative);
BytesAndMessagesCount consCounts1 = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Dispatch, ResourceGroupUsageStatsType.Cumulative);
Assert.assertEquals(prodCounts1.bytes, prodCounts.bytes);
Assert.assertEquals(prodCounts1.messages, prodCounts.messages);
Assert.assertEquals(consCounts1.bytes, consCounts.bytes);
Assert.assertEquals(consCounts1.messages, consCounts.messages);
if (checkProduce) {
Assert.assertTrue(prodCounts.bytes >= sentNumBytes);
Assert.assertEquals(sentNumMsgs, prodCounts.messages);
}
if (checkConsume) {
Assert.assertTrue(consCounts.bytes >= recvdNumBytes);
Assert.assertEquals(recvdNumMsgs, consCounts.messages);
}
}
}
}
}
Aggregations