Examples with Producer org.apache.pulsar.client.api.Producer used on opensource projects

Example 46 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testConcurrentConsumerReceiveWhileReconnect.

// This is to test that the flow control counter doesn't get corrupted while concurrent receives during
// reconnections
@Test(dataProvider = "batch")
public void testConcurrentConsumerReceiveWhileReconnect(int batchMessageDelayMs) throws Exception {
    final int recvQueueSize = 100;
    final int numConsumersThreads = 10;
    final ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(recvQueueSize);
    String subName = UUID.randomUUID().toString();
    final Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic7", subName, conf);
    ExecutorService executor = Executors.newCachedThreadPool();
    final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    // there will be 10 threads calling receive() from the same consumer and will block
    Thread.sleep(100);
    // we restart the broker to reconnect
    restartBroker();
    Thread.sleep(2000);
    // publish 100 messages so that the consumers blocked on receive() will now get the messages
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic7", producerConf);
    for (int i = 0; i < recvQueueSize; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(500);
    ConsumerImpl consumerImpl = (ConsumerImpl) consumer;
    // The available permits should be 10 and num messages in the queue should be 90
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
    barrier.reset();
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    Thread.sleep(100);
    // The available permits should be 20 and num messages in the queue should be 80
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads * 2);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - (numConsumersThreads * 2));
    // clear the queue
    while (true) {
        Message msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
    }
    // The available permits should be 0 and num messages in the queue should be 0
    Assert.assertEquals(consumerImpl.getAvailablePermits(), 0);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), 0);
    barrier.reset();
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    // we again make 10 threads call receive() and get blocked
    Thread.sleep(100);
    restartBroker();
    Thread.sleep(2000);
    // The available permits should be 10 and num messages in the queue should be 90
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
    consumer.close();
}

Example 47 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testAsyncProducerAndConsumerWithZeroQueueSize.

@Test(timeOut = 2000)
public void testAsyncProducerAndConsumerWithZeroQueueSize() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int totalMsg = 100;
    final Set<String> produceMsgs = Sets.newHashSet();
    final Set<String> consumeMsgs = Sets.newHashSet();
    final ProducerConfiguration producerConf = new ProducerConfiguration();
    final ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
    // produce message
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        produceMsgs.add(message);
    }
    log.info(" start receiving messages :");
    CountDownLatch latch = new CountDownLatch(totalMsg);
    // receive messages
    ExecutorService executor = Executors.newFixedThreadPool(1);
    receiveAsync(consumer, totalMsg, 0, latch, consumeMsgs, executor);
    latch.await();
    // verify message produced correctly
    assertEquals(produceMsgs.size(), totalMsg);
    // verify produced and consumed messages must be exactly same
    produceMsgs.removeAll(consumeMsgs);
    assertTrue(produceMsgs.isEmpty());
    producer.close();
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 48 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testAsyncProducerAndAsyncAck.

@Test(dataProvider = "batch")
public void testAsyncProducerAndAsyncAck(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic2", producerConf);
    List<Future<MessageId>> futures = Lists.newArrayList();
    // Asynchronously produce messages
    for (int i = 0; i < 10; i++) {
        final String message = "my-message-" + i;
        Future<MessageId> future = producer.sendAsync(message.getBytes());
        futures.add(future);
    }
    log.info("Waiting for async publish to complete");
    for (Future<MessageId> future : futures) {
        future.get();
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.info("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Asynchronously acknowledge upto and including the last message
    Future<Void> ackFuture = consumer.acknowledgeCumulativeAsync(msg);
    log.info("Waiting for async ack to complete");
    ackFuture.get();
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 49 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testSyncProducerAndConsumer.

@Test(dataProvider = "batch")
public void testSyncProducerAndConsumer(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 50 with Producer

use of org.apache.pulsar.client.api.Producer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testMessageListener.

@Test(dataProvider = "batch", timeOut = 100000)
public void testMessageListener(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    int numMessages = 100;
    final CountDownLatch latch = new CountDownLatch(numMessages);
    conf.setMessageListener((consumer, msg) -> {
        Assert.assertNotNull(msg, "Message cannot be null");
        String receivedMessage = new String(msg.getData());
        log.debug("Received message [{}] in the listener", receivedMessage);
        consumer.acknowledgeAsync(msg);
        latch.countDown();
    });
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic3", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic3", producerConf);
    List<Future<MessageId>> futures = Lists.newArrayList();
    // Asynchronously produce messages
    for (int i = 0; i < numMessages; i++) {
        final String message = "my-message-" + i;
        Future<MessageId> future = producer.sendAsync(message.getBytes());
        futures.add(future);
    }
    log.info("Waiting for async publish to complete");
    for (Future<MessageId> future : futures) {
        future.get();
    }
    log.info("Waiting for message listener to ack all messages");
    assertEquals(latch.await(numMessages, TimeUnit.SECONDS), true, "Timed out waiting for message listener acks");
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}