Examples with PersistentTopicStats org.apache.pulsar.common.policies.data.PersistentTopicStats used on opensource projects

Example 16 with PersistentTopicStats

use of org.apache.pulsar.common.policies.data.PersistentTopicStats in project incubator-pulsar by apache.

the class BrokerServiceTest method testBrokerServicePersistentRedeliverTopicStats.

@Test
public void testBrokerServicePersistentRedeliverTopicStats() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/successSharedTopic";
    final String subName = "successSharedSub";
    PersistentTopicStats stats;
    SubscriptionStats subStats;
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Shared).subscribe();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    // subscription stats
    assertEquals(stats.subscriptions.keySet().size(), 1);
    assertEquals(subStats.msgBacklog, 0);
    assertEquals(subStats.consumers.size(), 1);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    // publisher stats
    assertEquals(subStats.msgBacklog, 10);
    assertEquals(stats.publishers.size(), 1);
    assertTrue(stats.publishers.get(0).msgRateIn > 0.0);
    assertTrue(stats.publishers.get(0).msgThroughputIn > 0.0);
    assertTrue(stats.publishers.get(0).averageMsgSize > 0.0);
    // aggregated publish stats
    assertEquals(stats.msgRateIn, stats.publishers.get(0).msgRateIn);
    assertEquals(stats.msgThroughputIn, stats.publishers.get(0).msgThroughputIn);
    double diff = stats.averageMsgSize - stats.publishers.get(0).averageMsgSize;
    assertTrue(Math.abs(diff) < 0.000001);
    // consumer stats
    assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
    assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
    assertEquals(subStats.msgRateRedeliver, 0.0);
    assertEquals(subStats.consumers.get(0).unackedMessages, 10);
    // aggregated consumer stats
    assertEquals(subStats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(subStats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(stats.msgRateOut, subStats.consumers.get(0).msgRateOut);
    assertEquals(stats.msgThroughputOut, subStats.consumers.get(0).msgThroughputOut);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    assertEquals(subStats.unackedMessages, subStats.consumers.get(0).unackedMessages);
    consumer.redeliverUnacknowledgedMessages();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    assertTrue(subStats.msgRateRedeliver > 0.0);
    assertEquals(subStats.msgRateRedeliver, subStats.consumers.get(0).msgRateRedeliver);
    Message<byte[]> msg;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    consumer.close();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    rolloverPerIntervalStats();
    stats = topicRef.getStats();
    subStats = stats.subscriptions.values().iterator().next();
    assertEquals(subStats.msgBacklog, 0);
}

Example 17 with PersistentTopicStats

use of org.apache.pulsar.common.policies.data.PersistentTopicStats in project incubator-pulsar by apache.

the class PeerReplicatorTest method testPeerClusterTopicLookup.

/**
 * It verifies that lookup/admin requests for global-namespace would be redirected to peer-cluster if local cluster
 * doesn't own it and peer-cluster owns it, else request will be failed.
 * <pre>
 * 1. Create global-namespace ns1 for replication cluster-r1
 * 2. Try to create producer using broker in cluster r3
 * 3. Reject lookup: "r3" receives request and doesn't find namespace in local/peer cluster
 * 4. Add "r1" as a peer-cluster into "r3"
 * 5. Try to create producer using broker in cluster r3
 * 6. Success : "r3" finds "r1" in peer cluster which owns n1 and redirects to "r1"
 * 7. call admin-api to "r3" which redirects request to "r1"
 *
 * </pre>
 *
 * @param protocol
 * @throws Exception
 */
@Test(dataProvider = "lookupType")
public void testPeerClusterTopicLookup(String protocol) throws Exception {
    final String serviceUrl = protocol.equalsIgnoreCase("http") ? pulsar3.getWebServiceAddress() : pulsar3.getBrokerServiceUrl();
    final String namespace1 = "pulsar/global/peer1-" + protocol;
    final String namespace2 = "pulsar/global/peer2-" + protocol;
    admin1.namespaces().createNamespace(namespace1);
    admin1.namespaces().createNamespace(namespace2);
    // add replication cluster
    admin1.namespaces().setNamespaceReplicationClusters(namespace1, Lists.newArrayList("r1"));
    admin1.namespaces().setNamespaceReplicationClusters(namespace2, Lists.newArrayList("r2"));
    admin1.clusters().updatePeerClusterNames("r3", null);
    // disable tls as redirection url is prepared according tls configuration
    pulsar1.getConfiguration().setTlsEnabled(false);
    pulsar2.getConfiguration().setTlsEnabled(false);
    pulsar3.getConfiguration().setTlsEnabled(false);
    final String topic1 = "persistent://" + namespace1 + "/topic1";
    final String topic2 = "persistent://" + namespace2 + "/topic2";
    PulsarClient client3 = PulsarClient.builder().serviceUrl(serviceUrl).statsInterval(0, TimeUnit.SECONDS).build();
    try {
        // try to create producer for topic1 (part of cluster: r1) by calling cluster: r3
        client3.newProducer().topic(topic1).create();
        fail("should have failed as cluster:r3 doesn't own namespace");
    } catch (PulsarClientException e) {
    // Ok
    }
    try {
        // try to create producer for topic2 (part of cluster: r2) by calling cluster: r3
        client3.newProducer().topic(topic2).create();
        fail("should have failed as cluster:r3 doesn't own namespace");
    } catch (PulsarClientException e) {
    // Ok
    }
    // set peer-clusters : r3->r1
    admin1.clusters().updatePeerClusterNames("r3", Sets.newLinkedHashSet(Lists.newArrayList("r1")));
    Producer<byte[]> producer = client3.newProducer().topic(topic1).create();
    PersistentTopic topic = (PersistentTopic) pulsar1.getBrokerService().getTopic(topic1).get();
    assertNotNull(topic);
    pulsar1.getBrokerService().updateRates();
    // get stats for topic1 using cluster-r3's admin3
    PersistentTopicStats stats = admin1.persistentTopics().getStats(topic1);
    assertNotNull(stats);
    assertEquals(stats.publishers.size(), 1);
    stats = admin3.persistentTopics().getStats(topic1);
    assertNotNull(stats);
    assertEquals(stats.publishers.size(), 1);
    producer.close();
    // set peer-clusters : r3->r2
    admin2.clusters().updatePeerClusterNames("r3", Sets.newLinkedHashSet(Lists.newArrayList("r2")));
    producer = client3.newProducer().topic(topic2).create();
    topic = (PersistentTopic) pulsar2.getBrokerService().getTopic(topic2).get();
    assertNotNull(topic);
    pulsar2.getBrokerService().updateRates();
    // get stats for topic1 using cluster-r3's admin3
    stats = admin3.persistentTopics().getStats(topic2);
    assertNotNull(stats);
    assertEquals(stats.publishers.size(), 1);
    stats = admin3.persistentTopics().getStats(topic2);
    assertNotNull(stats);
    assertEquals(stats.publishers.size(), 1);
    producer.close();
    client3.close();
}

Example 18 with PersistentTopicStats

use of org.apache.pulsar.common.policies.data.PersistentTopicStats in project incubator-pulsar by apache.

the class DispatcherBlockConsumerTest method testBlockDispatcherStats.

@Test
public void testBlockDispatcherStats() throws Exception {
    int orginalDispatcherLimit = conf.getMaxUnackedMessagesPerSubscription();
    try {
        final String topicName = "persistent://prop/use/ns-abc/blockDispatch";
        final String subName = "blockDispatch";
        final int timeWaitToSync = 100;
        PersistentTopicStats stats;
        SubscriptionStats subStats;
        // configure maxUnackMessagePerDispatcher then restart broker to get this change
        conf.setMaxUnackedMessagesPerSubscription(10);
        stopBroker();
        startBroker();
        Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Shared).subscribe();
        Thread.sleep(timeWaitToSync);
        PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
        assertNotNull(topicRef);
        rolloverPerIntervalStats();
        stats = topicRef.getStats();
        subStats = stats.subscriptions.values().iterator().next();
        // subscription stats
        assertEquals(stats.subscriptions.keySet().size(), 1);
        assertEquals(subStats.msgBacklog, 0);
        assertEquals(subStats.consumers.size(), 1);
        Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
        Thread.sleep(timeWaitToSync);
        for (int i = 0; i < 100; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        Thread.sleep(timeWaitToSync);
        rolloverPerIntervalStats();
        stats = topicRef.getStats();
        subStats = stats.subscriptions.values().iterator().next();
        assertTrue(subStats.msgBacklog > 0);
        assertTrue(subStats.unackedMessages > 0);
        assertTrue(subStats.blockedSubscriptionOnUnackedMsgs);
        assertEquals(subStats.consumers.get(0).unackedMessages, subStats.unackedMessages);
        // consumer stats
        assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
        assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
        assertEquals(subStats.msgRateRedeliver, 0.0);
        producer.close();
        consumer.close();
    } finally {
        conf.setMaxUnackedMessagesPerSubscription(orginalDispatcherLimit);
    }
}

Example 19 with PersistentTopicStats

use of org.apache.pulsar.common.policies.data.PersistentTopicStats in project incubator-pulsar by apache.

the class PersistentTopic method getStats.

public PersistentTopicStats getStats() {
    PersistentTopicStats stats = new PersistentTopicStats();
    ObjectObjectHashMap<String, PublisherStats> remotePublishersStats = new ObjectObjectHashMap<String, PublisherStats>();
    producers.forEach(producer -> {
        PublisherStats publisherStats = producer.getStats();
        stats.msgRateIn += publisherStats.msgRateIn;
        stats.msgThroughputIn += publisherStats.msgThroughputIn;
        if (producer.isRemote()) {
            remotePublishersStats.put(producer.getRemoteCluster(), publisherStats);
        } else {
            stats.publishers.add(publisherStats);
        }
    });
    stats.averageMsgSize = stats.msgRateIn == 0.0 ? 0.0 : (stats.msgThroughputIn / stats.msgRateIn);
    subscriptions.forEach((name, subscription) -> {
        SubscriptionStats subStats = subscription.getStats();
        stats.msgRateOut += subStats.msgRateOut;
        stats.msgThroughputOut += subStats.msgThroughputOut;
        stats.subscriptions.put(name, subStats);
    });
    replicators.forEach((cluster, replicator) -> {
        ReplicatorStats replicatorStats = replicator.getStats();
        // Add incoming msg rates
        PublisherStats pubStats = remotePublishersStats.get(replicator.getRemoteCluster());
        if (pubStats != null) {
            replicatorStats.msgRateIn = pubStats.msgRateIn;
            replicatorStats.msgThroughputIn = pubStats.msgThroughputIn;
            replicatorStats.inboundConnection = pubStats.address;
            replicatorStats.inboundConnectedSince = pubStats.connectedSince;
        }
        stats.msgRateOut += replicatorStats.msgRateOut;
        stats.msgThroughputOut += replicatorStats.msgThroughputOut;
        stats.replication.put(replicator.getRemoteCluster(), replicatorStats);
    });
    stats.storageSize = ledger.getEstimatedBacklogSize();
    stats.deduplicationStatus = messageDeduplication.getStatus().toString();
    return stats;
}

Example 20 with PersistentTopicStats

use of org.apache.pulsar.common.policies.data.PersistentTopicStats in project incubator-pulsar by apache.

the class AdminApiTest method testPersistentTopicsExpireMessages.

/**
 * <pre>
 * Verify: PersistentTopicsBase.expireMessages()/expireMessagesForAllSubscriptions()
 * 1. Created multiple shared subscriptions and publisher on topic
 * 2. Publish messages on the topic
 * 3. expire message on sub-1 : backlog for sub-1 must be 0
 * 4. expire message on all subscriptions: backlog for all subscription must be 0
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testPersistentTopicsExpireMessages() throws Exception {
    // Force to create a topic
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/ds2", 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList("persistent://prop-xyz/use/ns1/ds2"));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    ConsumerBuilder<byte[]> consumerBuilder = client.newConsumer().topic("persistent://prop-xyz/use/ns1/ds2").subscriptionType(SubscriptionType.Shared);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().subscriptionName("my-sub1").subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().subscriptionName("my-sub2").subscribe();
    Consumer<byte[]> consumer3 = consumerBuilder.clone().subscriptionName("my-sub3").subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1/ds2").size(), 3);
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/ds2", 10);
    PersistentTopicStats topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 10);
    // wait for 1 seconds to expire message
    Thread.sleep(1000);
    admin.persistentTopics().expireMessages("persistent://prop-xyz/use/ns1/ds2", "my-sub1", 1);
    // wait for 1 seconds to execute expire message as it is async
    Thread.sleep(1000);
    topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 10);
    admin.persistentTopics().expireMessagesForAllSubscriptions("persistent://prop-xyz/use/ns1/ds2", 1);
    // wait for 1 seconds to execute expire message as it is async
    Thread.sleep(1000);
    topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 0);
    consumer1.close();
    consumer2.close();
    consumer3.close();
}