Examples with PulsarClient org.apache.pulsar.client.api.PulsarClient used on opensource projects

Example 76 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class AdminApiTest method testPersistentTopicsExpireMessages.

/**
 * <pre>
 * Verify: PersistentTopicsBase.expireMessages()/expireMessagesForAllSubscriptions()
 * 1. Created multiple shared subscriptions and publisher on topic
 * 2. Publish messages on the topic
 * 3. expire message on sub-1 : backlog for sub-1 must be 0
 * 4. expire message on all subscriptions: backlog for all subscription must be 0
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testPersistentTopicsExpireMessages() throws Exception {
    // Force to create a topic
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/ds2", 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList("persistent://prop-xyz/use/ns1/ds2"));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    ConsumerBuilder<byte[]> consumerBuilder = client.newConsumer().topic("persistent://prop-xyz/use/ns1/ds2").subscriptionType(SubscriptionType.Shared);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().subscriptionName("my-sub1").subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().subscriptionName("my-sub2").subscribe();
    Consumer<byte[]> consumer3 = consumerBuilder.clone().subscriptionName("my-sub3").subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions("persistent://prop-xyz/use/ns1/ds2").size(), 3);
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/ds2", 10);
    PersistentTopicStats topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 10);
    // wait for 1 seconds to expire message
    Thread.sleep(1000);
    admin.persistentTopics().expireMessages("persistent://prop-xyz/use/ns1/ds2", "my-sub1", 1);
    // wait for 1 seconds to execute expire message as it is async
    Thread.sleep(1000);
    topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 10);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 10);
    admin.persistentTopics().expireMessagesForAllSubscriptions("persistent://prop-xyz/use/ns1/ds2", 1);
    // wait for 1 seconds to execute expire message as it is async
    Thread.sleep(1000);
    topicStats = admin.persistentTopics().getStats("persistent://prop-xyz/use/ns1/ds2");
    assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 0);
    assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 0);
    consumer1.close();
    consumer2.close();
    consumer3.close();
}

Example 77 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class AdminApiTest method testPersistentTopicExpireMessageOnParitionTopic.

/**
 * Verify: PersistentTopicsBase.expireMessages()/expireMessagesForAllSubscriptions() for PartitionTopic
 *
 * @throws Exception
 */
@Test
public void testPersistentTopicExpireMessageOnParitionTopic() throws Exception {
    admin.persistentTopics().createPartitionedTopic("persistent://prop-xyz/use/ns1/ds1", 4);
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Consumer<byte[]> consumer = client.newConsumer().topic("persistent://prop-xyz/use/ns1/ds1").subscriptionName("my-sub").subscribe();
    Producer<byte[]> producer = client.newProducer().topic("persistent://prop-xyz/use/ns1/ds1").messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
    for (int i = 0; i < 10; i++) {
        String message = "message-" + i;
        producer.send(message.getBytes());
    }
    PartitionedTopicStats topicStats = admin.persistentTopics().getPartitionedStats("persistent://prop-xyz/use/ns1/ds1", true);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
    PersistentTopicStats partitionStatsPartition0 = topicStats.partitions.get("persistent://prop-xyz/use/ns1/ds1-partition-0");
    PersistentTopicStats partitionStatsPartition1 = topicStats.partitions.get("persistent://prop-xyz/use/ns1/ds1-partition-1");
    assertEquals(partitionStatsPartition0.subscriptions.get("my-sub").msgBacklog, 3, 1);
    assertEquals(partitionStatsPartition1.subscriptions.get("my-sub").msgBacklog, 3, 1);
    Thread.sleep(1000);
    admin.persistentTopics().expireMessagesForAllSubscriptions("persistent://prop-xyz/use/ns1/ds1", 1);
    Thread.sleep(1000);
    topicStats = admin.persistentTopics().getPartitionedStats("persistent://prop-xyz/use/ns1/ds1", true);
    partitionStatsPartition0 = topicStats.partitions.get("persistent://prop-xyz/use/ns1/ds1-partition-0");
    partitionStatsPartition1 = topicStats.partitions.get("persistent://prop-xyz/use/ns1/ds1-partition-1");
    assertEquals(partitionStatsPartition0.subscriptions.get("my-sub").msgBacklog, 0);
    assertEquals(partitionStatsPartition1.subscriptions.get("my-sub").msgBacklog, 0);
    producer.close();
    consumer.close();
    client.close();
}

Example 78 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class AdminApiTest method partitionedTopics.

@Test(dataProvider = "topicName")
public void partitionedTopics(String topicName) throws Exception {
    assertEquals(admin.persistentTopics().getPartitionedTopicList("prop-xyz/use/ns1"), Lists.newArrayList());
    final String partitionedTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
    admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 4);
    assertEquals(admin.persistentTopics().getPartitionedTopicList("prop-xyz/use/ns1"), Lists.newArrayList(partitionedTopicName));
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 4);
    // check if the virtual topic doesn't get created
    List<String> topics = admin.persistentTopics().getList("prop-xyz/use/ns1");
    assertEquals(topics.size(), 0);
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata("persistent://prop-xyz/use/ns1/ds2").partitions, 0);
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Consumer<byte[]> consumer = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
    try {
        admin.persistentTopics().deleteSubscription(partitionedTopicName, "my-sub");
        fail("should have failed");
    } catch (PulsarAdminException.PreconditionFailedException e) {
    // ok
    } catch (Exception e) {
        fail(e.getMessage());
    }
    Consumer<byte[]> consumer1 = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub-1").subscribe();
    assertEquals(Sets.newHashSet(admin.persistentTopics().getSubscriptions(partitionedTopicName)), Sets.newHashSet("my-sub", "my-sub-1"));
    consumer1.close();
    admin.persistentTopics().deleteSubscription(partitionedTopicName, "my-sub-1");
    assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
    Producer<byte[]> producer = client.newProducer().topic(partitionedTopicName).messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
    for (int i = 0; i < 10; i++) {
        String message = "message-" + i;
        producer.send(message.getBytes());
    }
    assertEquals(Sets.newHashSet(admin.persistentTopics().getList("prop-xyz/use/ns1")), Sets.newHashSet(partitionedTopicName + "-partition-0", partitionedTopicName + "-partition-1", partitionedTopicName + "-partition-2", partitionedTopicName + "-partition-3"));
    // test cumulative stats for partitioned topic
    PartitionedTopicStats topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, false);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
    assertEquals(topicStats.publishers.size(), 1);
    assertEquals(topicStats.partitions, Maps.newHashMap());
    // test per partition stats for partitioned topic
    topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, true);
    assertEquals(topicStats.metadata.partitions, 4);
    assertEquals(topicStats.partitions.keySet(), Sets.newHashSet(partitionedTopicName + "-partition-0", partitionedTopicName + "-partition-1", partitionedTopicName + "-partition-2", partitionedTopicName + "-partition-3"));
    PersistentTopicStats partitionStats = topicStats.partitions.get(partitionedTopicName + "-partition-0");
    assertEquals(partitionStats.publishers.size(), 1);
    assertEquals(partitionStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(partitionStats.subscriptions.get("my-sub").msgBacklog, 3, 1);
    try {
        admin.persistentTopics().skipMessages(partitionedTopicName, "my-sub", 5);
        fail("skip messages for partitioned topics should fail");
    } catch (Exception e) {
    // ok
    }
    admin.persistentTopics().skipAllMessages(partitionedTopicName, "my-sub");
    topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, false);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
    producer.close();
    consumer.close();
    admin.persistentTopics().deleteSubscription(partitionedTopicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList());
    try {
        admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 32);
        fail("Should have failed as the partitioned topic already exists");
    } catch (ConflictException ce) {
    }
    producer = client.newProducer().topic(partitionedTopicName).create();
    topics = admin.persistentTopics().getList("prop-xyz/use/ns1");
    assertEquals(topics.size(), 4);
    try {
        admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
        fail("The topic is busy");
    } catch (PreconditionFailedException pfe) {
    // ok
    }
    producer.close();
    client.close();
    admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 0);
    admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 32);
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 32);
    try {
        admin.persistentTopics().deletePartitionedTopic("persistent://prop-xyz/use/ns1/ds2");
        fail("Should have failed as the partitioned topic was not created");
    } catch (NotFoundException nfe) {
    }
    admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
    // delete a partitioned topic in a global namespace
    admin.persistentTopics().createPartitionedTopic(partitionedTopicName, 4);
    admin.persistentTopics().deletePartitionedTopic(partitionedTopicName);
}

Example 79 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class AdminApiTest method persistentTopicsInvalidCursorReset.

@Test
public void persistentTopicsInvalidCursorReset() throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    String topicName = "persistent://prop-xyz/use/ns1/invalidcursorreset";
    // Force to create a topic
    publishMessagesOnPersistentTopic(topicName, 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList(topicName));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Consumer<byte[]> consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, 10);
    List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);
    for (int i = 0; i < 10; i++) {
        Message<byte[]> message = consumer.receive();
        consumer.acknowledge(message);
    }
    // use invalid timestamp
    try {
        admin.persistentTopics().resetCursor(topicName, "my-sub", System.currentTimeMillis() - 190000);
    } catch (Exception e) {
        // fail the test
        throw e;
    }
    admin.persistentTopics().resetCursor(topicName, "my-sub", System.currentTimeMillis() + 90000);
    consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    consumer.close();
    client.close();
    admin.persistentTopics().deleteSubscription(topicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList());
    admin.persistentTopics().delete(topicName);
}

Example 80 with PulsarClient

use of org.apache.pulsar.client.api.PulsarClient in project incubator-pulsar by apache.

the class AdminApiTest2 method testIncrementPartitionsOfTopic.

/**
 * <pre>
 * It verifies increasing partitions for partitioned-topic.
 * 1. create a partitioned-topic
 * 2. update partitions with larger number of partitions
 * 3. verify: getPartitionedMetadata and check number of partitions
 * 4. verify: this api creates existing subscription to new partitioned-topics
 *            so, message will not be lost in new partitions
 *  a. start producer and produce messages
 *  b. check existing subscription for new topics and it should have backlog msgs
 *
 * </pre>
 *
 * @param topicName
 * @throws Exception
 */
@Test
public void testIncrementPartitionsOfTopic() throws Exception {
    final String topicName = "increment-partitionedTopic";
    final String subName1 = topicName + "-my-sub-1";
    final String subName2 = topicName + "-my-sub-2";
    final int startPartitions = 4;
    final int newPartitions = 8;
    final String partitionedTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    admin.persistentTopics().createPartitionedTopic(partitionedTopicName, startPartitions);
    // validate partition topic is created
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, startPartitions);
    // create consumer and subscriptions : check subscriptions
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).build();
    Consumer<byte[]> consumer1 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName1).subscriptionType(SubscriptionType.Shared).subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions(partitionedTopicName), Lists.newArrayList(subName1));
    Consumer<byte[]> consumer2 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName2).subscriptionType(SubscriptionType.Shared).subscribe();
    assertEquals(Sets.newHashSet(admin.persistentTopics().getSubscriptions(partitionedTopicName)), Sets.newHashSet(subName1, subName2));
    // (1) update partitions
    admin.persistentTopics().updatePartitionedTopic(partitionedTopicName, newPartitions);
    // invalidate global-cache to make sure that mock-zk-cache reds fresh data
    pulsar.getGlobalZkCache().invalidateAll();
    // verify new partitions have been created
    assertEquals(admin.persistentTopics().getPartitionedTopicMetadata(partitionedTopicName).partitions, newPartitions);
    // (2) No Msg loss: verify new partitions have the same existing subscription names
    final String newPartitionTopicName = TopicName.get(partitionedTopicName).getPartition(startPartitions + 1).toString();
    // (3) produce messages to all partitions including newly created partitions (RoundRobin)
    Producer<byte[]> producer = client.newProducer().topic(partitionedTopicName).messageRoutingMode(MessageRoutingMode.RoundRobinPartition).create();
    final int totalMessages = newPartitions * 2;
    for (int i = 0; i < totalMessages; i++) {
        String message = "message-" + i;
        producer.send(message.getBytes());
    }
    // (4) verify existing subscription has not lost any message: create new consumer with sub-2: it will load all
    // newly created partition topics
    consumer2.close();
    consumer2 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName2).subscriptionType(SubscriptionType.Shared).subscribe();
    // sometime: mockZk fails to refresh ml-cache: so, invalidate the cache to get fresh data
    pulsar.getLocalZkCacheService().managedLedgerListCache().clearTree();
    assertEquals(Sets.newHashSet(admin.persistentTopics().getSubscriptions(newPartitionTopicName)), Sets.newHashSet(subName1, subName2));
    assertEquals(Sets.newHashSet(admin.persistentTopics().getList("prop-xyz/use/ns1")).size(), newPartitions);
    // test cumulative stats for partitioned topic
    PartitionedTopicStats topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, false);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList(subName1, subName2)));
    assertEquals(topicStats.subscriptions.get(subName2).consumers.size(), 1);
    assertEquals(topicStats.subscriptions.get(subName2).msgBacklog, totalMessages);
    assertEquals(topicStats.publishers.size(), 1);
    assertEquals(topicStats.partitions, Maps.newHashMap());
    // (5) verify: each partition should have backlog
    topicStats = admin.persistentTopics().getPartitionedStats(partitionedTopicName, true);
    assertEquals(topicStats.metadata.partitions, newPartitions);
    Set<String> partitionSet = Sets.newHashSet();
    for (int i = 0; i < newPartitions; i++) {
        partitionSet.add(partitionedTopicName + "-partition-" + i);
    }
    assertEquals(topicStats.partitions.keySet(), partitionSet);
    for (int i = 0; i < newPartitions; i++) {
        PersistentTopicStats partitionStats = topicStats.partitions.get(TopicName.get(partitionedTopicName).getPartition(i).toString());
        assertEquals(partitionStats.publishers.size(), 1);
        assertEquals(partitionStats.subscriptions.get(subName2).consumers.size(), 1);
        assertEquals(partitionStats.subscriptions.get(subName2).msgBacklog, 2, 1);
    }
    producer.close();
    consumer1.close();
    consumer2.close();
    consumer2.close();
}