Examples with PersistentTopicStats org.apache.pulsar.common.policies.data.PersistentTopicStats used on opensource projects

Example 21 with PersistentTopicStats

use of org.apache.pulsar.common.policies.data.PersistentTopicStats in project incubator-pulsar by apache.

the class AdminApiTest method testPersistentTopicExpireMessageOnParitionTopic.

/**
 * Verify: PersistentTopicsBase.expireMessages()/expireMessagesForAllSubscriptions() for PartitionTopic
 *
 * @throws Exception
 */
@Test
public void testPersistentTopicExpireMessageOnParitionTopic() throws Exception {
    admin.persistentTopics().createPartitionedTopic("persistent://prop-xyz/use/ns1/ds1", 4);
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Consumer<byte[]> consumer = client.newConsumer().topic("persistent://prop-xyz/use/ns1/ds1").subscriptionName("my-sub").subscribe();
    Producer<byte[]> producer = client.newProducer().topic("persistent://prop-xyz/use/ns1/ds1").messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
    for (int i = 0; i < 10; i++) {
        String message = "message-" + i;
        producer.send(message.getBytes());
    }
    PartitionedTopicStats topicStats = admin.persistentTopics().getPartitionedStats("persistent://prop-xyz/use/ns1/ds1", true);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
    PersistentTopicStats partitionStatsPartition0 = topicStats.partitions.get("persistent://prop-xyz/use/ns1/ds1-partition-0");
    PersistentTopicStats partitionStatsPartition1 = topicStats.partitions.get("persistent://prop-xyz/use/ns1/ds1-partition-1");
    assertEquals(partitionStatsPartition0.subscriptions.get("my-sub").msgBacklog, 3, 1);
    assertEquals(partitionStatsPartition1.subscriptions.get("my-sub").msgBacklog, 3, 1);
    Thread.sleep(1000);
    admin.persistentTopics().expireMessagesForAllSubscriptions("persistent://prop-xyz/use/ns1/ds1", 1);
    Thread.sleep(1000);
    topicStats = admin.persistentTopics().getPartitionedStats("persistent://prop-xyz/use/ns1/ds1", true);
    partitionStatsPartition0 = topicStats.partitions.get("persistent://prop-xyz/use/ns1/ds1-partition-0");
    partitionStatsPartition1 = topicStats.partitions.get("persistent://prop-xyz/use/ns1/ds1-partition-1");
    assertEquals(partitionStatsPartition0.subscriptions.get("my-sub").msgBacklog, 0);
    assertEquals(partitionStatsPartition1.subscriptions.get("my-sub").msgBacklog, 0);
    producer.close();
    consumer.close();
    client.close();
}

Example 22 with PersistentTopicStats

use of org.apache.pulsar.common.policies.data.PersistentTopicStats in project incubator-pulsar by apache.

the class AdminApiTest method testPulsarAdminForUriAndUrlEncoding.

/**
 * This test-case verifies that broker should support both url/uri encoding for topic-name. It calls below api with
 * url-encoded and also uri-encoded topic-name in http request: a. PartitionedMetadataLookup b. TopicLookup c. Topic
 * Stats
 *
 * @param topicName
 * @throws Exception
 */
@Test(dataProvider = "topicName")
public void testPulsarAdminForUriAndUrlEncoding(String topicName) throws Exception {
    final String ns1 = "prop-xyz/use/ns1";
    final String topic1 = "persistent://" + ns1 + "/" + topicName;
    final String urlEncodedTopic = Codec.encode(topicName);
    final String uriEncodedTopic = urlEncodedTopic.replaceAll("\\+", "%20");
    final int numOfPartitions = 4;
    admin.persistentTopics().createPartitionedTopic(topic1, numOfPartitions);
    // Create a consumer to get stats on this topic
    pulsarClient.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
    PersistentTopicsImpl persistent = (PersistentTopicsImpl) admin.persistentTopics();
    Field field = PersistentTopicsImpl.class.getDeclaredField("adminPersistentTopics");
    field.setAccessible(true);
    WebTarget persistentTopics = (WebTarget) field.get(persistent);
    // (1) Get PartitionedMetadata : with Url and Uri encoding
    final CompletableFuture<PartitionedTopicMetadata> urlEncodedPartitionedMetadata = new CompletableFuture<>();
    // (a) Url encoding
    persistent.asyncGetRequest(persistentTopics.path(ns1).path(urlEncodedTopic).path("partitions"), new InvocationCallback<PartitionedTopicMetadata>() {

        @Override
        public void completed(PartitionedTopicMetadata response) {
            urlEncodedPartitionedMetadata.complete(response);
        }

        @Override
        public void failed(Throwable e) {
            Assert.fail(e.getMessage());
        }
    });
    final CompletableFuture<PartitionedTopicMetadata> uriEncodedPartitionedMetadata = new CompletableFuture<>();
    // (b) Uri encoding
    persistent.asyncGetRequest(persistentTopics.path(ns1).path(uriEncodedTopic).path("partitions"), new InvocationCallback<PartitionedTopicMetadata>() {

        @Override
        public void completed(PartitionedTopicMetadata response) {
            uriEncodedPartitionedMetadata.complete(response);
        }

        @Override
        public void failed(Throwable e) {
            uriEncodedPartitionedMetadata.completeExceptionally(e);
        }
    });
    assertEquals(urlEncodedPartitionedMetadata.get().partitions, numOfPartitions);
    assertEquals(urlEncodedPartitionedMetadata.get().partitions, (uriEncodedPartitionedMetadata.get().partitions));
    // (2) Get Topic Lookup
    LookupImpl lookup = (LookupImpl) admin.lookups();
    Field field2 = LookupImpl.class.getDeclaredField("v2lookup");
    field2.setAccessible(true);
    WebTarget target2 = (WebTarget) field2.get(lookup);
    // (a) Url encoding
    LookupData urlEncodedLookupData = lookup.request(target2.path("/destination/persistent").path(ns1 + "/" + urlEncodedTopic)).get(LookupData.class);
    // (b) Uri encoding
    LookupData uriEncodedLookupData = lookup.request(target2.path("/destination/persistent").path(ns1 + "/" + uriEncodedTopic)).get(LookupData.class);
    Assert.assertNotNull(urlEncodedLookupData.getBrokerUrl());
    assertEquals(urlEncodedLookupData.getBrokerUrl(), uriEncodedLookupData.getBrokerUrl());
    // (3) Get Topic Stats
    final CompletableFuture<PersistentTopicStats> urlStats = new CompletableFuture<>();
    // (a) Url encoding
    persistent.asyncGetRequest(persistentTopics.path(ns1).path(urlEncodedTopic + "-partition-1").path("stats"), new InvocationCallback<PersistentTopicStats>() {

        @Override
        public void completed(PersistentTopicStats response) {
            urlStats.complete(response);
        }

        @Override
        public void failed(Throwable e) {
            urlStats.completeExceptionally(e);
        }
    });
    // (b) Uri encoding
    final CompletableFuture<PersistentTopicStats> uriStats = new CompletableFuture<>();
    persistent.asyncGetRequest(persistentTopics.path(ns1).path(uriEncodedTopic + "-partition-1").path("stats"), new InvocationCallback<PersistentTopicStats>() {

        @Override
        public void completed(PersistentTopicStats response) {
            uriStats.complete(response);
        }

        @Override
        public void failed(Throwable e) {
            uriStats.completeExceptionally(e);
        }
    });
    assertEquals(urlStats.get().subscriptions.size(), 1);
    assertEquals(uriStats.get().subscriptions.size(), 1);
}

Example 23 with PersistentTopicStats

use of org.apache.pulsar.common.policies.data.PersistentTopicStats in project incubator-pulsar by apache.

the class AdminApiTest method partitionedTopics.

@Test(dataProvider = "topicName")
public void partitionedTopics(String topicName) throws Exception {
    assertEquals(admin.persistentTopics().getPartitionedTopicList("prop-xyz/use/ns1"), Lists.newArrayList());
    final String partitionedTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
    admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 4);
    assertEquals(admin.persistentTopics().getPartitionedTopicList("prop-xyz/use/ns1"), Lists.newArrayList(partitionedTopicName));
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 4);
    // check if the virtual topic doesn't get created
    List<String> topics = admin.persistentTopics().getList("prop-xyz/use/ns1");
    assertEquals(topics.size(), 0);
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata("persistent://prop-xyz/use/ns1/ds2").partitions, 0);
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Consumer<byte[]> consumer = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
    try {
        admin.persistentTopics().deleteSubscription(partitionedTopicName, "my-sub");
        fail("should have failed");
    } catch (PulsarAdminException.PreconditionFailedException e) {
    // ok
    } catch (Exception e) {
        fail(e.getMessage());
    }
    Consumer<byte[]> consumer1 = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub-1").subscribe();
    assertEquals(Sets.newHashSet(admin.persistentTopics().getSubscriptions(partitionedTopicName)), Sets.newHashSet("my-sub", "my-sub-1"));
    consumer1.close();
    admin.persistentTopics().deleteSubscription(partitionedTopicName, "my-sub-1");
    assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
    Producer<byte[]> producer = client.newProducer().topic(partitionedTopicName).messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
    for (int i = 0; i < 10; i++) {
        String message = "message-" + i;
        producer.send(message.getBytes());
    }
    assertEquals(Sets.newHashSet(admin.persistentTopics().getList("prop-xyz/use/ns1")), Sets.newHashSet(partitionedTopicName + "-partition-0", partitionedTopicName + "-partition-1", partitionedTopicName + "-partition-2", partitionedTopicName + "-partition-3"));
    // test cumulative stats for partitioned topic
    PartitionedTopicStats topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, false);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
    assertEquals(topicStats.publishers.size(), 1);
    assertEquals(topicStats.partitions, Maps.newHashMap());
    // test per partition stats for partitioned topic
    topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, true);
    assertEquals(topicStats.metadata.partitions, 4);
    assertEquals(topicStats.partitions.keySet(), Sets.newHashSet(partitionedTopicName + "-partition-0", partitionedTopicName + "-partition-1", partitionedTopicName + "-partition-2", partitionedTopicName + "-partition-3"));
    PersistentTopicStats partitionStats = topicStats.partitions.get(partitionedTopicName + "-partition-0");
    assertEquals(partitionStats.publishers.size(), 1);
    assertEquals(partitionStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(partitionStats.subscriptions.get("my-sub").msgBacklog, 3, 1);
    try {
        admin.persistentTopics().skipMessages(partitionedTopicName, "my-sub", 5);
        fail("skip messages for partitioned topics should fail");
    } catch (Exception e) {
    // ok
    }
    admin.persistentTopics().skipAllMessages(partitionedTopicName, "my-sub");
    topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, false);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
    producer.close();
    consumer.close();
    admin.persistentTopics().deleteSubscription(partitionedTopicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList());
    try {
        admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 32);
        fail("Should have failed as the partitioned topic already exists");
    } catch (ConflictException ce) {
    }
    producer = client.newProducer().topic(partitionedTopicName).create();
    topics = admin.persistentTopics().getList("prop-xyz/use/ns1");
    assertEquals(topics.size(), 4);
    try {
        admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
        fail("The topic is busy");
    } catch (PreconditionFailedException pfe) {
    // ok
    }
    producer.close();
    client.close();
    admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 0);
    admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 32);
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 32);
    try {
        admin.persistentTopics().deletePartitionedTopic("persistent://prop-xyz/use/ns1/ds2");
        fail("Should have failed as the partitioned topic was not created");
    } catch (NotFoundException nfe) {
    }
    admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
    // delete a partitioned topic in a global namespace
    admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 4);
    admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
}

Example 24 with PersistentTopicStats

use of org.apache.pulsar.common.policies.data.PersistentTopicStats in project incubator-pulsar by apache.

the class AdminApiTest2 method testIncrementPartitionsOfTopic.

/**
 * <pre>
 * It verifies increasing partitions for partitioned-topic.
 * 1. create a partitioned-topic
 * 2. update partitions with larger number of partitions
 * 3. verify: getPartitionedMetadata and check number of partitions
 * 4. verify: this api creates existing subscription to new partitioned-topics
 *            so, message will not be lost in new partitions
 *  a. start producer and produce messages
 *  b. check existing subscription for new topics and it should have backlog msgs
 *
 * </pre>
 *
 * @param topicName
 * @throws Exception
 */
@Test
public void testIncrementPartitionsOfTopic() throws Exception {
    final String topicName = "increment-partitionedTopic";
    final String subName1 = topicName + "-my-sub-1";
    final String subName2 = topicName + "-my-sub-2";
    final int startPartitions = 4;
    final int newPartitions = 8;
    final String partitionedTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    admin.persistentTopics().createPartitionedTopic(partitionedTopicName, startPartitions);
    // validate partition topic is created
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, startPartitions);
    // create consumer and subscriptions : check subscriptions
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).build();
    Consumer<byte[]> consumer1 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName1).subscriptionType(SubscriptionType.Shared).subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList(subName1));
    Consumer<byte[]> consumer2 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName2).subscriptionType(SubscriptionType.Shared).subscribe();
    assertEquals(Sets.newHashSet(admin.persistentTopics().getSubscriptions(partitionedTopicName)), Sets.newHashSet(subName1, subName2));
    // (1) update partitions
    admin.persistentTopics().updatePartitionedTopic(partitionedTopicName, newPartitions);
    // invalidate global-cache to make sure that mock-zk-cache reds fresh data
    pulsar.getGlobalZkCache().invalidateAll();
    // verify new partitions have been created
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, newPartitions);
    // (2) No Msg loss: verify new partitions have the same existing subscription names
    final String newPartitionTopicName = TopicName.get(partitionedTopicName).getPartition(startPartitions + 1).toString();
    // (3) produce messages to all partitions including newly created partitions (RoundRobin)
    Producer<byte[]> producer = client.newProducer().topic(partitionedTopicName).messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
    final int totalMessages = newPartitions * 2;
    for (int i = 0; i < totalMessages; i++) {
        String message = "message-" + i;
        producer.send(message.getBytes());
    }
    // (4) verify existing subscription has not lost any message: create new consumer with sub-2: it will load all
    // newly created partition topics
    consumer2.close();
    consumer2 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName2).subscriptionType(SubscriptionType.Shared).subscribe();
    // sometime: mockZk fails to refresh ml-cache: so, invalidate the cache to get fresh data
    pulsar.getLocalZkCacheService().managedLedgerListCache().clearTree();
    assertEquals(Sets.newHashSet(admin.persistentTopics().getSubscriptions(newPartitionTopicName)), Sets.newHashSet(subName1, subName2));
    assertEquals(Sets.newHashSet(admin.persistentTopics().getList("prop-xyz/use/ns1")).size(), newPartitions);
    // test cumulative stats for partitioned topic
    PartitionedTopicStats topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, false);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList(subName1, subName2)));
    assertEquals(topicStats.subscriptions.get(subName2).consumers.size(), 1);
    assertEquals(topicStats.subscriptions.get(subName2).msgBacklog, totalMessages);
    assertEquals(topicStats.publishers.size(), 1);
    assertEquals(topicStats.partitions, Maps.newHashMap());
    // (5) verify: each partition should have backlog
    topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, true);
    assertEquals(topicStats.metadata.partitions, newPartitions);
    Set<String> partitionSet = Sets.newHashSet();
    for (int i = 0; i < newPartitions; i++) {
        partitionSet.add(partitionedTopicName + "-partition-" + i);
    }
    assertEquals(topicStats.partitions.keySet(), partitionSet);
    for (int i = 0; i < newPartitions; i++) {
        PersistentTopicStats partitionStats = topicStats.partitions.get(TopicName.get(partitionedTopicName).getPartition(i).toString());
        assertEquals(partitionStats.publishers.size(), 1);
        assertEquals(partitionStats.subscriptions.get(subName2).consumers.size(), 1);
        assertEquals(partitionStats.subscriptions.get(subName2).msgBacklog, 2, 1);
    }
    producer.close();
    consumer1.close();
    consumer2.close();
    consumer2.close();
}

Example 25 with PersistentTopicStats

use of org.apache.pulsar.common.policies.data.PersistentTopicStats in project incubator-pulsar by apache.

the class BacklogQuotaManagerTest method testEvictionMulti.

@Test
public void testEvictionMulti() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(15 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    final String topic1 = "persistent://prop/usc/ns-quota/topic14";
    final String subName1 = "c14";
    final String subName2 = "c24";
    final int numMsgs = 10;
    final CyclicBarrier barrier = new CyclicBarrier(4);
    final CountDownLatch counter = new CountDownLatch(4);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    final PulsarClient client = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final Consumer<byte[]> consumer1 = client.newConsumer().topic(topic1).subscriptionName(subName1).subscribe();
    final Consumer<byte[]> consumer2 = client.newConsumer().topic(topic1).subscriptionName(subName2).subscribe();
    final PulsarClient client3 = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    final PulsarClient client2 = PulsarClient.builder().serviceUrl(adminUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Thread producerThread1 = new Thread() {

        public void run() {
            try {
                barrier.await();
                org.apache.pulsar.client.api.Producer<byte[]> producer = client2.newProducer().topic(topic1).create();
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread producerThread2 = new Thread() {

        public void run() {
            try {
                barrier.await();
                org.apache.pulsar.client.api.Producer<byte[]> producer = client3.newProducer().topic(topic1).create();
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread1 = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs * 2; i++) {
                    consumer1.acknowledge(consumer1.receive());
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread2 = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs * 2; i++) {
                    consumer2.acknowledge(consumer2.receive());
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    producerThread1.start();
    producerThread2.start();
    ConsumerThread1.start();
    ConsumerThread2.start();
    counter.await(20, TimeUnit.SECONDS);
    assertTrue(!gotException.get());
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertTrue(stats.storageSize <= 15 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
    client2.close();
    client3.close();
}