Examples with Consumer com.yahoo.pulsar.client.api.Consumer used on opensource projects

Example 86 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class PersistentTopicE2ETest method testConsumerFlowControl.

@Test
public void testConsumerFlowControl() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic2";
    final String subName = "sub2";
    Message msg;
    int recvQueueSize = 4;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    conf.setReceiverQueueSize(recvQueueSize);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertNotNull(subRef);
    // 1. initial receive queue size recorded
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), recvQueueSize);
    for (int i = 0; i < recvQueueSize / 2; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    // 2. queue size re-adjusted after successful receive of half of window size
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), recvQueueSize);
    consumer.close();
    assertFalse(subRef.getDispatcher().isConsumerConnected());
}

Example 87 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class PersistentTopicE2ETest method testSimpleCloseTopic.

@Test
public void testSimpleCloseTopic() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic5";
    final String subName = "sub5";
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, subName, conf);
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    PersistentSubscription subRef = topicRef.getPersistentSubscription(subName);
    assertNotNull(subRef);
    Message msg;
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        msg = consumer.receive();
        consumer.acknowledge(msg);
    }
    producer.close();
    consumer.close();
    topicRef.close().get();
    assertNull(pulsar.getBrokerService().getTopicReference(topicName));
}

Example 88 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testNoEviction.

@Test
public void testNoEviction() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/ns-quota"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/ns-quota", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.consumer_backlog_eviction));
    final String topic1 = "persistent://prop/usc/ns-quota/topic13";
    final String subName1 = "c13";
    final String subName2 = "c23";
    final int numMsgs = 10;
    final CyclicBarrier barrier = new CyclicBarrier(2);
    final CountDownLatch counter = new CountDownLatch(2);
    final AtomicBoolean gotException = new AtomicBoolean(false);
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final Consumer consumer1 = client.subscribe(topic1, subName1);
    final Consumer consumer2 = client.subscribe(topic1, subName2);
    final PulsarClient client2 = PulsarClient.create(adminUrl.toString(), clientConf);
    Thread producerThread = new Thread() {

        public void run() {
            try {
                barrier.await();
                com.yahoo.pulsar.client.api.Producer producer = client2.createProducer(topic1);
                byte[] content = new byte[1024];
                for (int i = 0; i < numMsgs; i++) {
                    producer.send(content);
                }
                producer.close();
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    Thread ConsumerThread = new Thread() {

        public void run() {
            try {
                barrier.await();
                for (int i = 0; i < numMsgs; i++) {
                    consumer1.acknowledge(consumer1.receive());
                    consumer2.acknowledge(consumer2.receive());
                }
            } catch (Exception e) {
                gotException.set(true);
            } finally {
                counter.countDown();
            }
        }
    };
    producerThread.start();
    ConsumerThread.start();
    counter.await();
    assertTrue(!gotException.get());
    client.close();
    client2.close();
}

Example 89 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class BacklogQuotaManagerTest method testAheadProducerOnHold.

@Test
public void testAheadProducerOnHold() throws Exception {
    assertEquals(admin.namespaces().getBacklogQuotaMap("prop/usc/quotahold"), Maps.newTreeMap());
    admin.namespaces().setBacklogQuota("prop/usc/quotahold", new BacklogQuota(10 * 1024, BacklogQuota.RetentionPolicy.producer_request_hold));
    final ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    final PulsarClient client = PulsarClient.create(adminUrl.toString(), clientConf);
    final String topic1 = "persistent://prop/usc/quotahold/hold";
    final String subName1 = "c1hold";
    final int numMsgs = 10;
    Consumer consumer = client.subscribe(topic1, subName1);
    ProducerConfiguration producerConfiguration = new ProducerConfiguration();
    producerConfiguration.setSendTimeout(2, TimeUnit.SECONDS);
    byte[] content = new byte[1024];
    Producer producer = client.createProducer(topic1, producerConfiguration);
    for (int i = 0; i <= numMsgs; i++) {
        try {
            producer.send(content);
            LOG.info("sent [{}]", i);
        } catch (PulsarClientException.TimeoutException cte) {
            // producer close may cause a timeout on send
            LOG.info("timeout on [{}]", i);
        }
    }
    for (int i = 0; i < numMsgs; i++) {
        consumer.receive();
        LOG.info("received [{}]", i);
    }
    Thread.sleep((TIME_TO_CHECK_BACKLOG_QUOTA + 1) * 1000);
    rolloverStats();
    PersistentTopicStats stats = admin.persistentTopics().getStats(topic1);
    Assert.assertEquals(stats.publishers.size(), 0, "Number of producers on topic " + topic1 + " are [" + stats.publishers.size() + "]");
    client.close();
}

Example 90 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class BatchMessageTest method testNonBatchCumulativeAckAfterBatchPublish.

@Test
public void testNonBatchCumulativeAckAfterBatchPublish() throws Exception {
    int numMsgs = 10;
    int numMsgsInBatch = numMsgs;
    final String topicName = "persistent://prop/use/ns-abc/testNonBatchCumulativeAckAfterBatchPublish";
    final String subscriptionName = "nbcaabp-sub-1";
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    // create producer to publish non batch messages
    Producer noBatchProducer = pulsarClient.createProducer(topicName);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    sendFutureList.clear();
    byte[] nobatchmsg = ("nobatch").getBytes();
    Message nmsg = MessageBuilder.create().setContent(nobatchmsg).build();
    noBatchProducer.sendAsync(nmsg).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 2);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    Message lastunackedMsg = null;
    for (int i = 0; i <= numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        lastunackedMsg = msg;
    }
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    rolloverPerIntervalStats();
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    assertTrue(((ConsumerImpl) consumer).isBatchingAckTrackerEmpty());
    consumer.close();
    producer.close();
    noBatchProducer.close();
}