Search in sources :

Example 1 with TopicStatsImpl

use of org.apache.pulsar.common.policies.data.stats.TopicStatsImpl in project pulsar by apache.

the class PersistentTopicsBase method internalGetPartitionedStats.

protected void internalGetPartitionedStats(AsyncResponse asyncResponse, boolean authoritative, boolean perPartition, boolean getPreciseBacklog, boolean subscriptionBacklogSize, boolean getEarliestTimeInBacklog) {
    CompletableFuture<Void> future;
    if (topicName.isGlobal()) {
        future = validateGlobalNamespaceOwnershipAsync(namespaceName);
    } else {
        future = CompletableFuture.completedFuture(null);
    }
    future.thenCompose(__ -> getPartitionedTopicMetadataAsync(topicName, authoritative, false)).thenAccept(partitionMetadata -> {
        if (partitionMetadata.partitions == 0) {
            asyncResponse.resume(new RestException(Status.NOT_FOUND, "Partitioned Topic not found"));
            return;
        }
        PartitionedTopicStatsImpl stats = new PartitionedTopicStatsImpl(partitionMetadata);
        List<CompletableFuture<TopicStats>> topicStatsFutureList = Lists.newArrayList();
        for (int i = 0; i < partitionMetadata.partitions; i++) {
            try {
                topicStatsFutureList.add(pulsar().getAdminClient().topics().getStatsAsync((topicName.getPartition(i).toString()), getPreciseBacklog, subscriptionBacklogSize, getEarliestTimeInBacklog));
            } catch (PulsarServerException e) {
                asyncResponse.resume(new RestException(e));
                return;
            }
        }
        FutureUtil.waitForAll(topicStatsFutureList).handle((result, exception) -> {
            CompletableFuture<TopicStats> statFuture = null;
            for (int i = 0; i < topicStatsFutureList.size(); i++) {
                statFuture = topicStatsFutureList.get(i);
                if (statFuture.isDone() && !statFuture.isCompletedExceptionally()) {
                    try {
                        stats.add(statFuture.get());
                        if (perPartition) {
                            stats.getPartitions().put(topicName.getPartition(i).toString(), (TopicStatsImpl) statFuture.get());
                        }
                    } catch (Exception e) {
                        asyncResponse.resume(new RestException(e));
                        return null;
                    }
                }
            }
            if (perPartition && stats.partitions.isEmpty()) {
                try {
                    boolean pathExists = namespaceResources().getPartitionedTopicResources().partitionedTopicExists(topicName);
                    if (pathExists) {
                        stats.partitions.put(topicName.toString(), new TopicStatsImpl());
                    } else {
                        asyncResponse.resume(new RestException(Status.NOT_FOUND, "Internal topics have not been generated yet"));
                        return null;
                    }
                } catch (Exception e) {
                    asyncResponse.resume(new RestException(e));
                    return null;
                }
            }
            asyncResponse.resume(stats);
            return null;
        });
    }).exceptionally(ex -> {
        // If the exception is not redirect exception we need to log it.
        if (!isRedirectException(ex)) {
            log.error("[{}] Failed to get partitioned internal stats for {}", clientAppId(), topicName, ex);
        }
        resumeAsyncResponseExceptionally(asyncResponse, ex);
        return null;
    });
}
Also used : Version(com.github.zafarkhaja.semver.Version) SubscriptionStats(org.apache.pulsar.common.policies.data.SubscriptionStats) ManagedLedgerOfflineBacklog(org.apache.bookkeeper.mledger.impl.ManagedLedgerOfflineBacklog) Topic(org.apache.pulsar.broker.service.Topic) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) NotFoundException(org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException) LedgerOffloader(org.apache.bookkeeper.mledger.LedgerOffloader) DelayedDeliveryPolicies(org.apache.pulsar.common.policies.data.DelayedDeliveryPolicies) StringUtils(org.apache.commons.lang3.StringUtils) SubscribeRate(org.apache.pulsar.common.policies.data.SubscribeRate) AlreadyRunningException(org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException) TopicOperation(org.apache.pulsar.common.policies.data.TopicOperation) PolicyOperation(org.apache.pulsar.common.policies.data.PolicyOperation) Map(java.util.Map) RestException(org.apache.pulsar.broker.web.RestException) NamespaceOperation(org.apache.pulsar.common.policies.data.NamespaceOperation) DispatchRateImpl(org.apache.pulsar.common.policies.data.impl.DispatchRateImpl) PersistentOfflineTopicStats(org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats) PositionImpl(org.apache.bookkeeper.mledger.impl.PositionImpl) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) PersistencePolicies(org.apache.pulsar.common.policies.data.PersistencePolicies) AsyncResponse(javax.ws.rs.container.AsyncResponse) EventsTopicNames.checkTopicIsTransactionCoordinatorAssign(org.apache.pulsar.common.events.EventsTopicNames.checkTopicIsTransactionCoordinatorAssign) Set(java.util.Set) PublishRate(org.apache.pulsar.common.policies.data.PublishRate) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) OffloadProcessStatus(org.apache.pulsar.client.admin.OffloadProcessStatus) MessageMetadata(org.apache.pulsar.common.api.proto.MessageMetadata) WebApplicationException(javax.ws.rs.WebApplicationException) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) AsyncCallbacks(org.apache.bookkeeper.mledger.AsyncCallbacks) Subscription(org.apache.pulsar.broker.service.Subscription) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) PulsarAdmin(org.apache.pulsar.client.admin.PulsarAdmin) PartitionedTopicMetadata(org.apache.pulsar.common.partition.PartitionedTopicMetadata) ArrayList(java.util.ArrayList) Commands(org.apache.pulsar.common.protocol.Commands) SubType(org.apache.pulsar.common.api.proto.CommandSubscribe.SubType) Lists(com.google.common.collect.Lists) DateFormatter(org.apache.pulsar.common.util.DateFormatter) BiConsumer(java.util.function.BiConsumer) NamespaceBundle(org.apache.pulsar.common.naming.NamespaceBundle) ResponseBuilder(javax.ws.rs.core.Response.ResponseBuilder) DispatchRate(org.apache.pulsar.common.policies.data.DispatchRate) InitialPosition(org.apache.pulsar.common.api.proto.CommandSubscribe.InitialPosition) ManagedLedgerInfo(org.apache.bookkeeper.mledger.ManagedLedgerInfo) IOException(java.io.IOException) PulsarService(org.apache.pulsar.broker.PulsarService) SubscriptionType(org.apache.pulsar.client.api.SubscriptionType) ExecutionException(java.util.concurrent.ExecutionException) PulsarService.isTransactionInternalName(org.apache.pulsar.broker.PulsarService.isTransactionInternalName) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) SubscriptionInvalidCursorPosition(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionInvalidCursorPosition) AuthorizationService(org.apache.pulsar.broker.authorization.AuthorizationService) DEFAULT_OPERATION_TIMEOUT_SEC(org.apache.pulsar.broker.resources.PulsarResources.DEFAULT_OPERATION_TIMEOUT_SEC) AdminResource(org.apache.pulsar.broker.admin.AdminResource) LoggerFactory(org.slf4j.LoggerFactory) KeyValue(org.apache.pulsar.common.api.proto.KeyValue) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) ManagedLedgerFactoryImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerFactoryImpl) MessageImpl(org.apache.pulsar.client.impl.MessageImpl) PreconditionFailedException(org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) ManagedLedger(org.apache.bookkeeper.mledger.ManagedLedger) ManagedLedgerConfig(org.apache.bookkeeper.mledger.ManagedLedgerConfig) AuthAction(org.apache.pulsar.common.policies.data.AuthAction) AuthPolicies(org.apache.pulsar.common.policies.data.AuthPolicies) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) BrokerEntryMetadata(org.apache.pulsar.common.api.proto.BrokerEntryMetadata) InactiveTopicPolicies(org.apache.pulsar.common.policies.data.InactiveTopicPolicies) PartitionedTopicInternalStats(org.apache.pulsar.common.policies.data.PartitionedTopicInternalStats) SchemaCompatibilityStrategy(org.apache.pulsar.common.policies.data.SchemaCompatibilityStrategy) OffloadPoliciesImpl(org.apache.pulsar.common.policies.data.OffloadPoliciesImpl) BitSetRecyclable(org.apache.pulsar.common.util.collections.BitSetRecyclable) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) CompletionException(java.util.concurrent.CompletionException) StreamingOutput(javax.ws.rs.core.StreamingOutput) Collectors(java.util.stream.Collectors) Sets(com.google.common.collect.Sets) AuthenticationDataSource(org.apache.pulsar.broker.authentication.AuthenticationDataSource) PulsarByteBufAllocator(org.apache.pulsar.common.allocator.PulsarByteBufAllocator) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) BacklogQuotaImpl(org.apache.pulsar.common.policies.data.impl.BacklogQuotaImpl) Base64(java.util.Base64) List(java.util.List) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) FutureUtil(org.apache.pulsar.common.util.FutureUtil) Response(javax.ws.rs.core.Response) EncryptionKeys(org.apache.pulsar.common.api.proto.EncryptionKeys) PersistentSubscription(org.apache.pulsar.broker.service.persistent.PersistentSubscription) PolicyName(org.apache.pulsar.common.policies.data.PolicyName) PartitionedTopicStatsImpl(org.apache.pulsar.common.policies.data.stats.PartitionedTopicStatsImpl) Optional(java.util.Optional) IntStream(java.util.stream.IntStream) CompressionCodec(org.apache.pulsar.common.compression.CompressionCodec) TopicName(org.apache.pulsar.common.naming.TopicName) Entry(org.apache.bookkeeper.mledger.Entry) ManagedLedgerInfoCallback(org.apache.bookkeeper.mledger.AsyncCallbacks.ManagedLedgerInfoCallback) BacklogQuota(org.apache.pulsar.common.policies.data.BacklogQuota) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) AtomicReference(java.util.concurrent.atomic.AtomicReference) ByteBuf(io.netty.buffer.ByteBuf) LongRunningProcessStatus(org.apache.pulsar.client.admin.LongRunningProcessStatus) CompressionCodecProvider(org.apache.pulsar.common.compression.CompressionCodecProvider) TopicDomain(org.apache.pulsar.common.naming.TopicDomain) Status(javax.ws.rs.core.Response.Status) PartitionedManagedLedgerInfo(org.apache.pulsar.common.naming.PartitionedManagedLedgerInfo) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) RetentionPolicies(org.apache.pulsar.common.policies.data.RetentionPolicies) PersistentReplicator(org.apache.pulsar.broker.service.persistent.PersistentReplicator) Logger(org.slf4j.Logger) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) TopicPolicies(org.apache.pulsar.common.policies.data.TopicPolicies) Maps(com.google.common.collect.Maps) TimeUnit(java.util.concurrent.TimeUnit) Policies(org.apache.pulsar.common.policies.data.Policies) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException) MessageId(org.apache.pulsar.client.api.MessageId) TransactionCoordinatorID(org.apache.pulsar.transaction.coordinator.TransactionCoordinatorID) Collections(java.util.Collections) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) PartitionedTopicStatsImpl(org.apache.pulsar.common.policies.data.stats.PartitionedTopicStatsImpl) CompletableFuture(java.util.concurrent.CompletableFuture) RestException(org.apache.pulsar.broker.web.RestException) PartitionedTopicStatsImpl(org.apache.pulsar.common.policies.data.stats.PartitionedTopicStatsImpl) ArrayList(java.util.ArrayList) List(java.util.List) PersistentOfflineTopicStats(org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) NotAllowedException(org.apache.pulsar.broker.service.BrokerServiceException.NotAllowedException) NotFoundException(org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException) AlreadyRunningException(org.apache.pulsar.broker.service.BrokerServiceException.AlreadyRunningException) RestException(org.apache.pulsar.broker.web.RestException) MetadataStoreException(org.apache.pulsar.metadata.api.MetadataStoreException) SubscriptionBusyException(org.apache.pulsar.broker.service.BrokerServiceException.SubscriptionBusyException) WebApplicationException(javax.ws.rs.WebApplicationException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) PulsarServerException(org.apache.pulsar.broker.PulsarServerException) PreconditionFailedException(org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) CompletionException(java.util.concurrent.CompletionException) ManagedLedgerException(org.apache.bookkeeper.mledger.ManagedLedgerException) PulsarAdminException(org.apache.pulsar.client.admin.PulsarAdminException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) TopicBusyException(org.apache.pulsar.broker.service.BrokerServiceException.TopicBusyException)

Example 2 with TopicStatsImpl

use of org.apache.pulsar.common.policies.data.stats.TopicStatsImpl in project pulsar by apache.

the class RGUsageMTAggrWaitForAllMsgsTest method verifyRGProdConsStats.

// Verify the app stats with what we see from the broker-service, and the resource-group (which in turn internally
// derives stats from the broker service)
private void verifyRGProdConsStats(String[] topicStrings, int sentNumBytes, int sentNumMsgs, int recvdNumBytes, int recvdNumMsgs, int scaleFactor, boolean checkProduce, boolean checkConsume) throws Exception {
    BrokerService bs = pulsar.getBrokerService();
    Map<String, TopicStatsImpl> topicStatsMap = bs.getTopicStats();
    log.debug("verifyProdConsStats: topicStatsMap has {} entries", topicStatsMap.size());
    // Pulsar runtime adds some additional bytes in the exchanges: a 45-byte per-message
    // metadata of some kind, plus more as the number of messages increases.
    // Hence the ">=" assertion with ExpectedNumBytesSent/Received in the following checks.
    final int ExpectedNumBytesSent = sentNumBytes + PER_MESSAGE_METADATA_OHEAD * sentNumMsgs;
    final int ExpectedNumBytesReceived = recvdNumBytes + PER_MESSAGE_METADATA_OHEAD * recvdNumMsgs;
    long totalOutMessages = 0, totalOutBytes = 0;
    long totalInMessages = 0, totalInBytes = 0;
    BytesAndMessagesCount totalTenantRGProdCounts = new BytesAndMessagesCount();
    BytesAndMessagesCount totalTenantRGConsCounts = new BytesAndMessagesCount();
    BytesAndMessagesCount totalNsRGProdCounts = new BytesAndMessagesCount();
    BytesAndMessagesCount totalNsRGConsCounts = new BytesAndMessagesCount();
    BytesAndMessagesCount prodCounts, consCounts;
    // Since the following walk is on topics, keep track of the RGs for which we have already gathered stats,
    // so that we do not double-accumulate stats if multiple topics refer to the same RG.
    HashSet<String> RGsWithPublishStatsGathered = new HashSet<>();
    HashSet<String> RGsWithDispatchStatsGathered = new HashSet<>();
    // Hack to ensure aggregator calculation without waiting for a period of aggregation.
    // [aggregateResourceGroupLocalUsages() is idempotent when there's no fresh traffic flowing.]
    this.rgservice.aggregateResourceGroupLocalUsages();
    for (Map.Entry<String, TopicStatsImpl> entry : topicStatsMap.entrySet()) {
        String mapTopicName = entry.getKey();
        if (Arrays.asList(topicStrings).contains(mapTopicName)) {
            TopicStats stats = entry.getValue();
            totalInMessages += stats.getMsgInCounter();
            totalInBytes += stats.getBytesInCounter();
            totalOutMessages += stats.getMsgOutCounter();
            totalOutBytes += stats.getBytesOutCounter();
            // we should see some produced mesgs on every topic.
            if (totalInMessages == 0) {
                log.warn("verifyProdConsStats: found no produced mesgs (msgInCounter) on topic {}", mapTopicName);
            }
            if (sentNumMsgs > 0 || recvdNumMsgs > 0) {
                TopicName topic = TopicName.get(mapTopicName);
                final String tenantRGName = TopicToTenantRGName(topic);
                if (!RGsWithPublishStatsGathered.contains(tenantRGName)) {
                    prodCounts = this.rgservice.getRGUsage(tenantRGName, ResourceGroupMonitoringClass.Publish, getCumulativeUsageStats);
                    totalTenantRGProdCounts = ResourceGroup.accumulateBMCount(totalTenantRGProdCounts, prodCounts);
                    RGsWithPublishStatsGathered.add(tenantRGName);
                }
                if (!RGsWithDispatchStatsGathered.contains(tenantRGName)) {
                    consCounts = this.rgservice.getRGUsage(tenantRGName, ResourceGroupMonitoringClass.Dispatch, getCumulativeUsageStats);
                    totalTenantRGConsCounts = ResourceGroup.accumulateBMCount(totalTenantRGConsCounts, consCounts);
                    RGsWithDispatchStatsGathered.add(tenantRGName);
                }
                final String nsRGName = TopicToNamespaceRGName(topic);
                // We will do the same here, to get the expected stats.
                if (tenantRGName.compareTo(nsRGName) != 0) {
                    if (!RGsWithPublishStatsGathered.contains(nsRGName)) {
                        prodCounts = this.rgservice.getRGUsage(nsRGName, ResourceGroupMonitoringClass.Publish, getCumulativeUsageStats);
                        totalNsRGProdCounts = ResourceGroup.accumulateBMCount(totalNsRGProdCounts, prodCounts);
                        RGsWithPublishStatsGathered.add(nsRGName);
                    }
                    if (!RGsWithDispatchStatsGathered.contains(nsRGName)) {
                        consCounts = this.rgservice.getRGUsage(nsRGName, ResourceGroupMonitoringClass.Dispatch, getCumulativeUsageStats);
                        totalNsRGConsCounts = ResourceGroup.accumulateBMCount(totalNsRGConsCounts, consCounts);
                        RGsWithDispatchStatsGathered.add(nsRGName);
                    }
                }
            }
        }
    }
    // Check that the accumulated totals tally up.
    if (checkConsume && checkProduce) {
        Assert.assertEquals(totalOutMessages, totalInMessages);
        Assert.assertEquals(totalOutBytes, totalInBytes);
    }
    if (checkProduce) {
        Assert.assertEquals(totalInMessages, sentNumMsgs);
        Assert.assertTrue(totalInBytes >= ExpectedNumBytesSent);
    }
    if (checkConsume) {
        Assert.assertEquals(totalOutMessages, recvdNumMsgs);
        Assert.assertTrue(totalOutBytes >= ExpectedNumBytesReceived);
    }
    if (checkProduce) {
        prodCounts = ResourceGroup.accumulateBMCount(totalTenantRGProdCounts, totalNsRGProdCounts);
        Assert.assertEquals(prodCounts.messages, sentNumMsgs * scaleFactor);
        Assert.assertTrue(prodCounts.bytes >= ExpectedNumBytesSent);
    }
    if (checkConsume) {
        consCounts = ResourceGroup.accumulateBMCount(totalTenantRGConsCounts, totalNsRGConsCounts);
        Assert.assertEquals(consCounts.messages, recvdNumMsgs * scaleFactor);
        Assert.assertTrue(consCounts.bytes >= ExpectedNumBytesReceived);
    }
}
Also used : TopicName(org.apache.pulsar.common.naming.TopicName) TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) BrokerService(org.apache.pulsar.broker.service.BrokerService) BytesAndMessagesCount(org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount) Map(java.util.Map) TopicStats(org.apache.pulsar.common.policies.data.TopicStats) HashSet(java.util.HashSet)

Example 3 with TopicStatsImpl

use of org.apache.pulsar.common.policies.data.stats.TopicStatsImpl in project pulsar by apache.

the class PersistentTopicStatsTest method testPersistentTopicStatsAggregationPartialProducerIsNotSupported.

@Test
public void testPersistentTopicStatsAggregationPartialProducerIsNotSupported() {
    TopicStatsImpl topicStats1 = new TopicStatsImpl();
    topicStats1.msgRateIn = 1;
    topicStats1.msgThroughputIn = 1;
    topicStats1.msgRateOut = 1;
    topicStats1.msgThroughputOut = 1;
    topicStats1.averageMsgSize = 1;
    topicStats1.storageSize = 1;
    final PublisherStatsImpl publisherStats1 = new PublisherStatsImpl();
    publisherStats1.setSupportsPartialProducer(false);
    publisherStats1.setProducerName("name1");
    topicStats1.addPublisher(publisherStats1);
    topicStats1.subscriptions.put("test_ns", new SubscriptionStatsImpl());
    topicStats1.replication.put("test_ns", new ReplicatorStatsImpl());
    TopicStatsImpl topicStats2 = new TopicStatsImpl();
    topicStats2.msgRateIn = 1;
    topicStats2.msgThroughputIn = 2;
    topicStats2.msgRateOut = 3;
    topicStats2.msgThroughputOut = 4;
    topicStats2.averageMsgSize = 5;
    topicStats2.storageSize = 6;
    final PublisherStatsImpl publisherStats2 = new PublisherStatsImpl();
    publisherStats2.setSupportsPartialProducer(false);
    publisherStats2.setProducerName("name1");
    topicStats2.addPublisher(publisherStats2);
    topicStats2.subscriptions.put("test_ns", new SubscriptionStatsImpl());
    topicStats2.replication.put("test_ns", new ReplicatorStatsImpl());
    TopicStatsImpl target = new TopicStatsImpl();
    target.add(topicStats1);
    target.add(topicStats2);
    assertEquals(target.msgRateIn, 2.0);
    assertEquals(target.msgThroughputIn, 3.0);
    assertEquals(target.msgRateOut, 4.0);
    assertEquals(target.msgThroughputOut, 5.0);
    assertEquals(target.averageMsgSize, 3.0);
    assertEquals(target.storageSize, 7);
    assertEquals(target.getPublishers().size(), 1);
    assertEquals(target.subscriptions.size(), 1);
    assertEquals(target.replication.size(), 1);
}
Also used : TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) Test(org.testng.annotations.Test)

Example 4 with TopicStatsImpl

use of org.apache.pulsar.common.policies.data.stats.TopicStatsImpl in project pulsar by apache.

the class PersistentTopicStatsTest method testPersistentTopicStatsAggregationByProducerName.

@Test
public void testPersistentTopicStatsAggregationByProducerName() {
    TopicStatsImpl topicStats1 = new TopicStatsImpl();
    topicStats1.msgRateIn = 1;
    topicStats1.msgThroughputIn = 1;
    topicStats1.msgRateOut = 1;
    topicStats1.msgThroughputOut = 1;
    topicStats1.averageMsgSize = 1;
    topicStats1.storageSize = 1;
    final PublisherStatsImpl publisherStats1 = new PublisherStatsImpl();
    publisherStats1.setSupportsPartialProducer(true);
    publisherStats1.msgRateIn = 1;
    publisherStats1.setProducerName("name1");
    topicStats1.addPublisher(publisherStats1);
    topicStats1.subscriptions.put("test_ns", new SubscriptionStatsImpl());
    topicStats1.replication.put("test_ns", new ReplicatorStatsImpl());
    TopicStatsImpl topicStats2 = new TopicStatsImpl();
    topicStats2.msgRateIn = 1;
    topicStats2.msgThroughputIn = 2;
    topicStats2.msgRateOut = 3;
    topicStats2.msgThroughputOut = 4;
    topicStats2.averageMsgSize = 5;
    topicStats2.storageSize = 6;
    final PublisherStatsImpl publisherStats2 = new PublisherStatsImpl();
    publisherStats2.setSupportsPartialProducer(true);
    publisherStats2.msgRateIn = 1;
    publisherStats2.setProducerName("name1");
    topicStats2.addPublisher(publisherStats2);
    topicStats2.subscriptions.put("test_ns", new SubscriptionStatsImpl());
    topicStats2.replication.put("test_ns", new ReplicatorStatsImpl());
    TopicStatsImpl topicStats3 = new TopicStatsImpl();
    topicStats3.msgRateIn = 0;
    topicStats3.msgThroughputIn = 0;
    topicStats3.msgRateOut = 0;
    topicStats3.msgThroughputOut = 0;
    topicStats3.averageMsgSize = 0;
    topicStats3.storageSize = 0;
    final PublisherStatsImpl publisherStats3 = new PublisherStatsImpl();
    publisherStats3.setSupportsPartialProducer(true);
    publisherStats3.msgRateIn = 1;
    publisherStats3.setProducerName("name2");
    topicStats3.addPublisher(publisherStats3);
    topicStats3.subscriptions.put("test_ns", new SubscriptionStatsImpl());
    topicStats3.replication.put("test_ns", new ReplicatorStatsImpl());
    TopicStatsImpl target = new TopicStatsImpl();
    target.add(topicStats1);
    target.add(topicStats2);
    target.add(topicStats3);
    assertEquals(target.msgRateIn, 2.0);
    assertEquals(target.msgThroughputIn, 3.0);
    assertEquals(target.msgRateOut, 4.0);
    assertEquals(target.msgThroughputOut, 5.0);
    assertEquals(target.averageMsgSize, 2.0);
    assertEquals(target.storageSize, 7);
    assertEquals(target.getPublishers().size(), 2);
    final Map<String, Double> expectedPublishersMap = Maps.newHashMap();
    expectedPublishersMap.put("name1", 2.0);
    expectedPublishersMap.put("name2", 1.0);
    assertEquals(target.getPublishers().stream().collect(Collectors.toMap(PublisherStats::getProducerName, e -> ((PublisherStatsImpl) e).msgRateIn)), expectedPublishersMap);
    assertEquals(target.subscriptions.size(), 1);
    assertEquals(target.replication.size(), 1);
}
Also used : TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) ReplicatorStatsImpl(org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl) SubscriptionStatsImpl(org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl) PublisherStatsImpl(org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl) Test(org.testng.annotations.Test)

Example 5 with TopicStatsImpl

use of org.apache.pulsar.common.policies.data.stats.TopicStatsImpl in project pulsar by apache.

the class ResourceGroupUsageAggregationTest method verifyStats.

// Verify the app stats with what we see from the broker-service, and the resource-group (which in turn internally
// derives stats from the broker service)
// There appears to be a 45-byte message header which is accounted in the stats, additionally to what the
// application-level sends/receives. Hence, the byte counts are a ">=" check, instead of an equality check.
private void verifyStats(String topicString, String rgName, int sentNumBytes, int sentNumMsgs, int recvdNumBytes, int recvdNumMsgs, boolean checkProduce, boolean checkConsume) throws InterruptedException, PulsarAdminException {
    BrokerService bs = pulsar.getBrokerService();
    Map<String, TopicStatsImpl> topicStatsMap = bs.getTopicStats();
    for (Map.Entry<String, TopicStatsImpl> entry : topicStatsMap.entrySet()) {
        String mapTopicName = entry.getKey();
        if (mapTopicName.equals(topicString)) {
            TopicStatsImpl stats = entry.getValue();
            if (checkProduce) {
                Assert.assertTrue(stats.bytesInCounter >= sentNumBytes);
                Assert.assertEquals(sentNumMsgs, stats.msgInCounter);
            }
            if (checkConsume) {
                Assert.assertTrue(stats.bytesOutCounter >= recvdNumBytes);
                Assert.assertEquals(recvdNumMsgs, stats.msgOutCounter);
            }
            if (sentNumMsgs > 0 || recvdNumMsgs > 0) {
                // hack to ensure aggregator calculation without waiting
                rgs.aggregateResourceGroupLocalUsages();
                BytesAndMessagesCount prodCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Publish, ResourceGroupUsageStatsType.Cumulative);
                BytesAndMessagesCount consCounts = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Dispatch, ResourceGroupUsageStatsType.Cumulative);
                // Re-do the getRGUsage.
                // The counts should be equal, since there wasn't any intervening traffic on TEST_PRODUCE_CONSUME_TOPIC.
                BytesAndMessagesCount prodCounts1 = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Publish, ResourceGroupUsageStatsType.Cumulative);
                BytesAndMessagesCount consCounts1 = rgs.getRGUsage(rgName, ResourceGroupMonitoringClass.Dispatch, ResourceGroupUsageStatsType.Cumulative);
                Assert.assertEquals(prodCounts1.bytes, prodCounts.bytes);
                Assert.assertEquals(prodCounts1.messages, prodCounts.messages);
                Assert.assertEquals(consCounts1.bytes, consCounts.bytes);
                Assert.assertEquals(consCounts1.messages, consCounts.messages);
                if (checkProduce) {
                    Assert.assertTrue(prodCounts.bytes >= sentNumBytes);
                    Assert.assertEquals(sentNumMsgs, prodCounts.messages);
                }
                if (checkConsume) {
                    Assert.assertTrue(consCounts.bytes >= recvdNumBytes);
                    Assert.assertEquals(recvdNumMsgs, consCounts.messages);
                }
            }
        }
    }
}
Also used : TopicStatsImpl(org.apache.pulsar.common.policies.data.stats.TopicStatsImpl) BrokerService(org.apache.pulsar.broker.service.BrokerService) Map(java.util.Map) BytesAndMessagesCount(org.apache.pulsar.broker.resourcegroup.ResourceGroup.BytesAndMessagesCount)

Aggregations

TopicStatsImpl (org.apache.pulsar.common.policies.data.stats.TopicStatsImpl)10 ReplicatorStatsImpl (org.apache.pulsar.common.policies.data.stats.ReplicatorStatsImpl)6 SubscriptionStatsImpl (org.apache.pulsar.common.policies.data.stats.SubscriptionStatsImpl)6 Map (java.util.Map)5 PublisherStatsImpl (org.apache.pulsar.common.policies.data.stats.PublisherStatsImpl)5 TopicName (org.apache.pulsar.common.naming.TopicName)4 Test (org.testng.annotations.Test)4 Optional (java.util.Optional)3 BrokerService (org.apache.pulsar.broker.service.BrokerService)3 Lists (com.google.common.collect.Lists)2 Maps (com.google.common.collect.Maps)2 Sets (com.google.common.collect.Sets)2 ByteBuf (io.netty.buffer.ByteBuf)2 FastThreadLocal (io.netty.util.concurrent.FastThreadLocal)2 ArrayList (java.util.ArrayList)2 Collections (java.util.Collections)2 List (java.util.List)2 Set (java.util.Set)2 CompletableFuture (java.util.concurrent.CompletableFuture)2 CompletionException (java.util.concurrent.CompletionException)2