Examples with PulsarClient com.yahoo.pulsar.client.api.PulsarClient used on opensource projects

Example 1 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class AdminApiTest method persistentTopics.

@Test(dataProvider = "topicName")
public void persistentTopics(String topicName) throws Exception {
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    final String persistentTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
    // Force to create a destination
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList("persistent://prop-xyz/use/ns1/" + topicName));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = client.subscribe(persistentTopicName, "my-sub", conf);
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 10);
    PersistentTopicStats topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
    assertEquals(topicStats.publishers.size(), 0);
    PersistentTopicInternalStats internalStats = admin.persistentTopics().getInternalStats(persistentTopicName);
    assertEquals(internalStats.cursors.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    List<Message> messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 3);
    assertEquals(messages.size(), 3);
    for (int i = 0; i < 3; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 15);
    assertEquals(messages.size(), 10);
    for (int i = 0; i < 10; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    admin.persistentTopics().skipMessages(persistentTopicName, "my-sub", 5);
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 5);
    admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
    consumer.close();
    client.close();
    admin.persistentTopics().deleteSubscription(persistentTopicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList());
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet());
    assertEquals(topicStats.publishers.size(), 0);
    try {
        admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    } catch (NotFoundException e) {
    }
    admin.persistentTopics().delete(persistentTopicName);
    try {
        admin.persistentTopics().delete(persistentTopicName);
        fail("Should have received 404");
    } catch (NotFoundException e) {
    }
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
}

Example 2 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testConcurrentAckAndEviction.

@Test
public void testConcurrentAckAndEviction() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    final String topic1 = "persistent://prop/usc/ns-quota/topic12";
    final String subName1 = "c12";
    final String subName2 = "c22";
    final int numMsgs = 20;
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    PulsarClient client2 = PulsarClient.create(adminUrl.toString(), clientConf);
    Consumer consumer1 = client2.subscribe(topic1, subName1);
    Consumer consumer2 = client2.subscribe(topic1, subName2);
    Thread producerThread = new Thread() {

        public void run() {
            try {
                barrier.await();
                final com.yahoo.pulsar.client.api.Producer producer = client.createProducer(topic1);
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs; i++) {
                    // only one consumer acknowledges the message
                    consumer1.acknowledge(consumer1.receive());
                    consumer2.receive();
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    producerThread.start();
    ConsumerThread.start();
    // test hangs without timeout since there is nothing to consume due to eviction
    counter.await(20, TimeUnit.SECONDS);
    assertTrue(!gotException.get());
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertTrue(stats.storageSize <= 10 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
    client2.close();
}

Example 3 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testEvictionMulti.

@Test
public void testEvictionMulti() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(15 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    final String topic1 = "persistent://prop/usc/ns-quota/topic14";
    final String subName1 = "c14";
    final String subName2 = "c24";
    final int numMsgs = 10;
    final CyclicBarrier barrier = new CyclicBarrier(4);
    final CountDownLatch counter = new CountDownLatch(4);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final Consumer consumer1 = client.subscribe(topic1, subName1);
    final Consumer consumer2 = client.subscribe(topic1, subName2);
    final PulsarClient client3 = PulsarClient.create(adminUrl.toString(), clientConf);
    final PulsarClient client2 = PulsarClient.create(adminUrl.toString(), clientConf);
    Thread producerThread1 = new Thread() {

        public void run() {
            try {
                barrier.await();
                com.yahoo.pulsar.client.api.Producer producer = client2.createProducer(topic1);
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread producerThread2 = new Thread() {

        public void run() {
            try {
                barrier.await();
                com.yahoo.pulsar.client.api.Producer producer = client3.createProducer(topic1);
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread1 = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs * 2; i++) {
                    consumer1.acknowledge(consumer1.receive());
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread2 = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs * 2; i++) {
                    consumer2.acknowledge(consumer2.receive());
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    producerThread1.start();
    producerThread2.start();
    ConsumerThread1.start();
    ConsumerThread2.start();
    counter.await(20, TimeUnit.SECONDS);
    assertTrue(!gotException.get());
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertTrue(stats.storageSize <= 15 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
    client2.close();
    client3.close();
}

Example 4 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testConsumerBacklogEviction.

@Test
public void testConsumerBacklogEviction() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final String topic1 = "persistent://prop/usc/ns-quota/topic1";
    final String subName1 = "c1";
    final String subName2 = "c2";
    final int numMsgs = 20;
    Consumer consumer1 = client.subscribe(topic1, subName1);
    Consumer consumer2 = client.subscribe(topic1, subName2);
    com.yahoo.pulsar.client.api.Producer producer = client.createProducer(topic1);
    byte[] content = new byte[1024];
    for (int i = 0; i < numMsgs; i++) {
        producer.send(content);
        consumer1.receive();
        consumer2.receive();
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertTrue(stats.storageSize < 10 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
}

Example 5 with PulsarClient

use of com.yahoo.pulsar.client.api.PulsarClient in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testConsumerBacklogEvictionWithAck.

@Test
public void testConsumerBacklogEvictionWithAck() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    PulsarClient client = PulsarClient.create(adminUrl.toString());
    final String topic1 = "persistent://prop/usc/ns-quota/topic11";
    final String subName1 = "c11";
    final String subName2 = "c21";
    final int numMsgs = 20;
    Consumer consumer1 = client.subscribe(topic1, subName1);
    Consumer consumer2 = client.subscribe(topic1, subName2);
    com.yahoo.pulsar.client.api.Producer producer = client.createProducer(topic1);
    byte[] content = new byte[1024];
    for (int i = 0; i < numMsgs; i++) {
        producer.send(content);
        // only one consumer acknowledges the message
        consumer1.acknowledge(consumer1.receive());
        consumer2.receive();
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertTrue(stats.storageSize <= 10 * 1024, "Storage size is [" + stats.storageSize + "]");
    client.close();
}