Examples with TopicStats org.apache.pulsar.common.policies.data.TopicStats used on opensource projects

Example 1 with TopicStats

use of org.apache.pulsar.common.policies.data.TopicStats in project starlight-for-kafka by datastax.

the class KafkaTopicConsumerManagerTest method verifyBacklogAndNumCursor.

// dump Topic Stats, mainly want to get and verify backlogSize.
private void verifyBacklogAndNumCursor(PersistentTopic persistentTopic, long expectedBacklog, int numCursor) throws Exception {
    AtomicLong backlog = new AtomicLong(0);
    AtomicInteger cursorCount = new AtomicInteger(0);
    retryStrategically(((test) -> {
        backlog.set(persistentTopic.getStats(true, true).backlogSize);
        return backlog.get() == expectedBacklog;
    }), 5, 200);
    if (log.isDebugEnabled()) {
        TopicStats topicStats = persistentTopic.getStats(true, true);
        log.info(" dump topicStats for topic : {}, storageSize: {}, backlogSize: {}, expected: {}", persistentTopic.getName(), topicStats.getStorageSize(), topicStats.getBacklogSize(), expectedBacklog);
        topicStats.getSubscriptions().forEach((subname, substats) -> {
            log.debug(" dump sub: subname - {}, activeConsumerName {}, " + "consumers {}, msgBacklog {}, unackedMessages {}.", subname, substats.getActiveConsumerName(), substats.getConsumers(), substats.getMsgBacklog(), substats.getUnackedMessages());
        });
    }
    persistentTopic.getManagedLedger().getCursors().forEach(cursor -> {
        if (log.isDebugEnabled()) {
            log.debug(" dump cursor: cursor - {}, durable: {}, numberEntryis: {}," + " readPosition: {}, markdeletePosition: {}", cursor.getName(), cursor.isDurable(), cursor.getNumberOfEntries(), cursor.getReadPosition(), cursor.getMarkDeletedPosition());
        }
        cursorCount.incrementAndGet();
    });
    // verify.
    assertEquals(backlog.get(), expectedBacklog);
    assertEquals(cursorCount.get(), numCursor);
}

Example 2 with TopicStats

use of org.apache.pulsar.common.policies.data.TopicStats in project pulsar by apache.

the class AdminApiTest method testGetStats.

@Test
public void testGetStats() throws Exception {
    final String topic = "persistent://prop-xyz/ns1/my-topic" + UUID.randomUUID().toString();
    admin.topics().createNonPartitionedTopic(topic);
    String subName = "my-sub";
    // create consumer and subscription
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topic).subscriptionName(subName).subscribe();
    TopicStats topicStats = admin.topics().getStats(topic, false, false, true);
    assertEquals(topicStats.getEarliestMsgPublishTimeInBacklogs(), 0);
    assertEquals(topicStats.getSubscriptions().get(subName).getEarliestMsgPublishTimeInBacklog(), 0);
    // publish several messages
    publishMessagesOnPersistentTopic(topic, 10);
    Thread.sleep(1000);
    topicStats = admin.topics().getStats(topic, false, false, true);
    assertTrue(topicStats.getEarliestMsgPublishTimeInBacklogs() > 0);
    assertTrue(topicStats.getSubscriptions().get(subName).getEarliestMsgPublishTimeInBacklog() > 0);
    for (int i = 0; i < 10; i++) {
        Message<byte[]> message = consumer.receive();
        consumer.acknowledge(message);
    }
    Thread.sleep(1000);
    topicStats = admin.topics().getStats(topic, false, false, true);
    assertEquals(topicStats.getEarliestMsgPublishTimeInBacklogs(), 0);
    assertEquals(topicStats.getSubscriptions().get(subName).getEarliestMsgPublishTimeInBacklog(), 0);
}

Example 3 with TopicStats

use of org.apache.pulsar.common.policies.data.TopicStats in project pulsar by apache.

the class AdminApiTest method partitionedTopics.

@Test(dataProvider = "topicNamesForAllTypes")
public void partitionedTopics(String topicType, String topicName) throws Exception {
    final String namespace = "prop-xyz/ns1";
    final String partitionedTopicName = topicType + "://" + namespace + "/" + topicName;
    final String anotherTopic = topicType + "://" + namespace + "/ds2";
    // TODO: there're some gaps between non-persistent topics and persistent topics, so some checks will be skipped
    // for non-persistent topics. After the gaps were filled, we can remove this check.
    final boolean isPersistent = topicType.equals(TopicDomain.persistent.value());
    assertEquals(admin.topics().getPartitionedTopicList(namespace), Lists.newArrayList());
    try {
        admin.topics().getPartitionedTopicMetadata(partitionedTopicName);
        fail("getPartitionedTopicMetadata of " + partitionedTopicName + " should not succeed");
    } catch (NotFoundException expected) {
    }
    admin.topics().createPartitionedTopic(partitionedTopicName, 4);
    assertEquals(admin.topics().getPartitionedTopicList(namespace), Lists.newArrayList(partitionedTopicName));
    assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 4);
    List<String> topics;
    if (isPersistent) {
        // TODO: for non-persistent topics getList will return 0
        topics = admin.topics().getList(namespace);
        assertEquals(topics.size(), 4);
    }
    try {
        admin.topics().getPartitionedTopicMetadata(anotherTopic);
        fail("getPartitionedTopicMetadata of " + anotherTopic + " should not succeed");
    } catch (NotFoundException expected) {
    }
    PartitionedTopicStats topicStats = admin.topics().getPartitionedStats(partitionedTopicName, false);
    // check the getPartitionedStats for PartitionedTopic returns only partitions metadata, and no partitions info
    assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, topicStats.getMetadata().partitions);
    assertEquals(topicStats.getPartitions().size(), 0);
    List<String> subscriptions = admin.topics().getSubscriptions(partitionedTopicName);
    assertEquals(subscriptions.size(), 0);
    // create consumer and subscription
    @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(pulsar.getWebServiceAddress()).statsInterval(0, TimeUnit.SECONDS).build();
    Consumer<byte[]> consumer = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscribe();
    assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
    try {
        if (isPersistent) {
            // TODO: for non-persistent topics, deleteSubscription might throw NotFoundException
            admin.topics().deleteSubscription(partitionedTopicName, "my-sub");
            // TODO: for non-persistent topics, deleteSubscription won't fail
            fail("should have failed");
        }
    } catch (PreconditionFailedException e) {
    // ok
    } catch (Exception e) {
        fail(e.getMessage());
    }
    Consumer<byte[]> consumer1 = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub-1").subscribe();
    if (isPersistent) {
        // TODO: for non-persistent topics, getSubscriptions will return a empty set
        assertEquals(Sets.newHashSet(admin.topics().getSubscriptions(partitionedTopicName)), Sets.newHashSet("my-sub", "my-sub-1"));
    }
    consumer1.close();
    if (isPersistent) {
        // TODO: for non-persistent topics, deleteSubscription might throw NotFoundException
        admin.topics().deleteSubscription(partitionedTopicName, "my-sub-1");
        // TODO: for non-persistent topics, getSubscriptions will return a empty set
        assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
    }
    Producer<byte[]> producer = client.newProducer(Schema.BYTES).topic(partitionedTopicName).enableBatching(false).messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
    for (int i = 0; i < 10; i++) {
        String message = "message-" + i;
        producer.send(message.getBytes());
    }
    assertEquals(Sets.newHashSet(admin.topics().getList(namespace)), Sets.newHashSet(partitionedTopicName + "-partition-0", partitionedTopicName + "-partition-1", partitionedTopicName + "-partition-2", partitionedTopicName + "-partition-3"));
    // test cumulative stats for partitioned topic
    topicStats = admin.topics().getPartitionedStats(partitionedTopicName, false);
    if (isPersistent) {
        // TODO: for non-persistent topics, the subscription doesn't exist
        assertEquals(topicStats.getSubscriptions().keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
        assertEquals(topicStats.getSubscriptions().get("my-sub").getConsumers().size(), 1);
        assertEquals(topicStats.getSubscriptions().get("my-sub").getMsgBacklog(), 10);
    }
    assertEquals(topicStats.getPublishers().size(), 1);
    assertEquals(topicStats.getPartitions(), Maps.newHashMap());
    // test per partition stats for partitioned topic
    topicStats = admin.topics().getPartitionedStats(partitionedTopicName, true);
    assertEquals(topicStats.getMetadata().partitions, 4);
    assertEquals(topicStats.getPartitions().keySet(), Sets.newHashSet(partitionedTopicName + "-partition-0", partitionedTopicName + "-partition-1", partitionedTopicName + "-partition-2", partitionedTopicName + "-partition-3"));
    TopicStats partitionStats = topicStats.getPartitions().get(partitionedTopicName + "-partition-0");
    assertEquals(partitionStats.getPublishers().size(), 1);
    if (isPersistent) {
        // TODO: for non-persistent topics, the subscription doesn't exist
        assertEquals(partitionStats.getSubscriptions().get("my-sub").getConsumers().size(), 1);
        assertEquals(partitionStats.getSubscriptions().get("my-sub").getMsgBacklog(), 3, 1);
    }
    try {
        admin.topics().skipMessages(partitionedTopicName, "my-sub", 5);
        fail("skip messages for partitioned topics should fail");
    } catch (Exception e) {
    // ok
    }
    if (isPersistent) {
        // TODO: for non-persistent topics, skilAllMessages will cause 500 internal error
        admin.topics().skipAllMessages(partitionedTopicName, "my-sub");
        topicStats = admin.topics().getPartitionedStats(partitionedTopicName, false);
        assertEquals(topicStats.getSubscriptions().get("my-sub").getMsgBacklog(), 0);
    }
    producer.close();
    consumer.close();
    if (isPersistent) {
        // TODO: for non-persistent topics, deleteSubscription might throw NotFoundException
        admin.topics().deleteSubscription(partitionedTopicName, "my-sub");
        assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList());
    }
    try {
        admin.topics().createPartitionedTopic(partitionedTopicName, 32);
        fail("Should have failed as the partitioned topic already exists");
    } catch (ConflictException ignore) {
    }
    producer = client.newProducer(Schema.BYTES).topic(partitionedTopicName).enableBatching(false).messageRoutingMode(MessageRoutingMode.SinglePartition).create();
    if (isPersistent) {
        // TODO: for non-persistent topics getList will return 0
        topics = admin.topics().getList(namespace);
        assertEquals(topics.size(), 4);
    }
    try {
        admin.topics().deletePartitionedTopic(partitionedTopicName);
        fail("The topic is busy");
    } catch (PreconditionFailedException pfe) {
    // ok
    }
    producer.close();
    client.close();
    admin.topics().deletePartitionedTopic(partitionedTopicName);
    try {
        admin.topics().getPartitionedTopicMetadata(partitionedTopicName);
        fail("getPartitionedTopicMetadata of " + partitionedTopicName + " should not succeed");
    } catch (NotFoundException expected) {
    }
    admin.topics().createPartitionedTopic(partitionedTopicName, 32);
    assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 32);
    try {
        admin.topics().deletePartitionedTopic(anotherTopic);
        fail("Should have failed as the partitioned topic was not created");
    } catch (NotFoundException nfe) {
    }
    admin.topics().deletePartitionedTopic(partitionedTopicName);
    // delete a partitioned topic in a global namespace
    admin.topics().createPartitionedTopic(partitionedTopicName, 4);
    admin.topics().deletePartitionedTopic(partitionedTopicName);
}

Example 4 with TopicStats

use of org.apache.pulsar.common.policies.data.TopicStats in project pulsar by apache.

the class AdminApiTest method testPartitionedTopicMsgDelayedAggregated.

@Test
public void testPartitionedTopicMsgDelayedAggregated() throws Exception {
    final String topic = "persistent://prop-xyz/ns1/testPartitionedTopicMsgDelayedAggregated-" + UUID.randomUUID().toString();
    final String subName = "my-sub";
    final int numPartitions = 2;
    conf.setSubscriptionRedeliveryTrackerEnabled(true);
    conf.setDelayedDeliveryEnabled(true);
    admin.topics().createPartitionedTopic(topic, numPartitions);
    for (int i = 0; i < 2; i++) {
        pulsarClient.newConsumer().topic(topic).subscriptionType(SubscriptionType.Shared).subscriptionName(subName).subscribe();
    }
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topic).enableBatching(false).create();
    final int messages = 100;
    for (int i = 0; i < messages; i++) {
        String msg = "Hello Pulsar - " + i;
        producer.send(msg.getBytes());
        producer.newMessage().deliverAfter(1L, TimeUnit.HOURS).value(msg.getBytes()).send();
    }
    PartitionedTopicStats partitionedTopicStats = admin.topics().getPartitionedStats(topic, false);
    Assert.assertNotNull(partitionedTopicStats);
    SubscriptionStats subStats = partitionedTopicStats.getSubscriptions().get(subName);
    Assert.assertNotNull(subStats);
    Assert.assertEquals(subStats.getMsgBacklog(), subStats.getMsgBacklogNoDelayed() + subStats.getMsgDelayed());
    partitionedTopicStats = admin.topics().getPartitionedStats(topic, true);
    Assert.assertNotNull(partitionedTopicStats);
    subStats = partitionedTopicStats.getSubscriptions().get(subName);
    Assert.assertNotNull(subStats);
    Assert.assertEquals(subStats.getMsgBacklog(), subStats.getMsgBacklogNoDelayed() + subStats.getMsgDelayed());
    Assert.assertNotNull(partitionedTopicStats.getPartitions());
    Assert.assertEquals(partitionedTopicStats.getPartitions().size(), numPartitions);
    long sumMsgBacklog = 0;
    long sumMsgBacklogNoDelayed = 0;
    long sumMsgDelayed = 0;
    for (TopicStats stats : partitionedTopicStats.getPartitions().values()) {
        Assert.assertNotNull(stats);
        SubscriptionStats partitionedSubStats = stats.getSubscriptions().get(subName);
        Assert.assertNotNull(partitionedSubStats);
        sumMsgBacklog += partitionedSubStats.getMsgBacklog();
        sumMsgBacklogNoDelayed += partitionedSubStats.getMsgBacklogNoDelayed();
        sumMsgDelayed += partitionedSubStats.getMsgDelayed();
    }
    Assert.assertEquals(sumMsgBacklog, sumMsgBacklogNoDelayed + sumMsgDelayed);
    Assert.assertEquals(sumMsgBacklog, subStats.getMsgBacklog());
    Assert.assertEquals(sumMsgBacklogNoDelayed, subStats.getMsgBacklogNoDelayed());
    Assert.assertEquals(sumMsgDelayed, subStats.getMsgDelayed());
}

Example 5 with TopicStats

use of org.apache.pulsar.common.policies.data.TopicStats in project pulsar by apache.

the class AdminApiTest method testPersistentTopicsExpireMessages.

/**
 * <pre>
 * Verify: PersistentTopicsBase.expireMessages()/expireMessagesForAllSubscriptions()
 * 1. Created multiple shared subscriptions and publisher on topic
 * 2. Publish messages on the topic
 * 3. expire message on sub-1 : backlog for sub-1 must be 0
 * 4. expire message on all subscriptions: backlog for all subscription must be 0
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testPersistentTopicsExpireMessages() throws Exception {
    // Force to create a topic
    publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/ds2", 0);
    assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList("persistent://prop-xyz/ns1/ds2"));
    // create consumer and subscription
    @Cleanup PulsarClient client = PulsarClient.builder().serviceUrl(pulsar.getWebServiceAddress()).statsInterval(0, TimeUnit.SECONDS).build();
    ConsumerBuilder<byte[]> consumerBuilder = client.newConsumer().topic("persistent://prop-xyz/ns1/ds2").subscriptionType(SubscriptionType.Shared);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().subscriptionName("my-sub1").subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().subscriptionName("my-sub2").subscribe();
    Consumer<byte[]> consumer3 = consumerBuilder.clone().subscriptionName("my-sub3").subscribe();
    assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1/ds2").size(), 3);
    List<MessageId> messageIds = publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/ds2", 10);
    TopicStats topicStats = admin.topics().getStats("persistent://prop-xyz/ns1/ds2");
    assertEquals(topicStats.getSubscriptions().get("my-sub1").getMsgBacklog(), 10);
    assertEquals(topicStats.getSubscriptions().get("my-sub2").getMsgBacklog(), 10);
    assertEquals(topicStats.getSubscriptions().get("my-sub3").getMsgBacklog(), 10);
    Thread.sleep(1000);
    admin.topics().expireMessages("persistent://prop-xyz/ns1/ds2", "my-sub1", 1);
    // Wait at most 2 seconds for sub1's message to expire.
    Awaitility.await().untilAsserted(() -> assertTrue(admin.topics().getStats("persistent://prop-xyz/ns1/ds2").getSubscriptions().get("my-sub1").getLastMarkDeleteAdvancedTimestamp() > 0L));
    topicStats = admin.topics().getStats("persistent://prop-xyz/ns1/ds2");
    SubscriptionStats subStats1 = topicStats.getSubscriptions().get("my-sub1");
    assertEquals(subStats1.getMsgBacklog(), 0);
    SubscriptionStats subStats2 = topicStats.getSubscriptions().get("my-sub2");
    assertEquals(subStats2.getMsgBacklog(), 10);
    assertEquals(subStats2.getLastMarkDeleteAdvancedTimestamp(), 0L);
    SubscriptionStats subStats3 = topicStats.getSubscriptions().get("my-sub3");
    assertEquals(subStats3.getMsgBacklog(), 10);
    assertEquals(subStats3.getLastMarkDeleteAdvancedTimestamp(), 0L);
    admin.topics().expireMessages("persistent://prop-xyz/ns1/ds2", "my-sub2", messageIds.get(4), false);
    // Wait at most 2 seconds for sub2's message to expire.
    Awaitility.await().untilAsserted(() -> assertTrue(admin.topics().getStats("persistent://prop-xyz/ns1/ds2").getSubscriptions().get("my-sub2").getLastMarkDeleteAdvancedTimestamp() > 0L));
    topicStats = admin.topics().getStats("persistent://prop-xyz/ns1/ds2");
    subStats1 = topicStats.getSubscriptions().get("my-sub1");
    assertEquals(subStats1.getMsgBacklog(), 0);
    assertTrue(subStats1.getLastMarkDeleteAdvancedTimestamp() > 0L);
    long sub2lastMarkDeleteAdvancedTimestamp = subStats1.getLastMarkDeleteAdvancedTimestamp();
    subStats2 = topicStats.getSubscriptions().get("my-sub2");
    assertEquals(subStats2.getMsgBacklog(), 5);
    subStats3 = topicStats.getSubscriptions().get("my-sub3");
    assertEquals(subStats3.getMsgBacklog(), 10);
    assertEquals(subStats3.getLastMarkDeleteAdvancedTimestamp(), 0L);
    try {
        admin.topics().expireMessagesForAllSubscriptions("persistent://prop-xyz/ns1/ds2", 1);
    } catch (Exception e) {
        // my-sub1 has no msg backlog, so expire message won't be issued on that subscription
        assertTrue(e.getMessage().startsWith("Expire message by timestamp not issued on topic"));
    }
    // Wait at most 2 seconds for sub3's message to expire.
    Awaitility.await().untilAsserted(() -> assertTrue(admin.topics().getStats("persistent://prop-xyz/ns1/ds2").getSubscriptions().get("my-sub3").getLastMarkDeleteAdvancedTimestamp() > 0L));
    topicStats = admin.topics().getStats("persistent://prop-xyz/ns1/ds2");
    subStats1 = topicStats.getSubscriptions().get("my-sub1");
    assertEquals(subStats1.getMsgBacklog(), 0);
    assertEquals(subStats1.getLastMarkDeleteAdvancedTimestamp(), subStats1.getLastMarkDeleteAdvancedTimestamp());
    // Wait at most 2 seconds for rest of sub2's message to expire.
    subStats2 = topicStats.getSubscriptions().get("my-sub2");
    assertEquals(subStats2.getMsgBacklog(), 0);
    assertTrue(subStats2.getLastMarkDeleteAdvancedTimestamp() > sub2lastMarkDeleteAdvancedTimestamp);
    subStats3 = topicStats.getSubscriptions().get("my-sub3");
    assertEquals(subStats3.getMsgBacklog(), 0);
    consumer1.close();
    consumer2.close();
    consumer3.close();
}