Examples with Consumer org.apache.pulsar.client.api.Consumer used on opensource projects

Example 6 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testAsyncProducerAndConsumer.

@Test(timeOut = 2000)
public void testAsyncProducerAndConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int totalMsg = 100;
    final Set<String> produceMsgs = Sets.newHashSet();
    final Set<String> consumeMsgs = Sets.newHashSet();
    final ProducerConfiguration producerConf = new ProducerConfiguration();
    final ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
    // produce message
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        produceMsgs.add(message);
    }
    log.info(" start receiving messages :");
    CountDownLatch latch = new CountDownLatch(totalMsg);
    // receive messages
    ExecutorService executor = Executors.newFixedThreadPool(1);
    receiveAsync(consumer, totalMsg, 0, latch, consumeMsgs, executor);
    latch.await();
    // verify message produced correctly
    assertEquals(produceMsgs.size(), totalMsg);
    // verify produced and consumed messages must be exactly same
    produceMsgs.removeAll(consumeMsgs);
    assertTrue(produceMsgs.isEmpty());
    producer.close();
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 7 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testBlockUnackConsumerAckByDifferentConsumer.

/**
 * Verify: Consumer2 sends ack of Consumer1 and consumer1 should be unblock if it is blocked due to unack-messages
 *
 * @param batchMessageDelayMs
 * @throws Exception
 */
@Test
public void testBlockUnackConsumerAckByDifferentConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int maxUnackedMessages = 20;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 100;
        int totalReceiveMessages = 0;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnackedMessages);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) Consumer1: consume without ack:
        // try to consume messages: but will be able to consume number of messages = maxUnackedMessages
        Message msg = null;
        List<Message> messages = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer1.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                totalReceiveMessages++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // consumer1
        assertEquals(messages.size(), maxUnackedMessages);
        // (3) ack for all UnackedMessages from consumer2
        messages.forEach(m -> {
            try {
                consumer2.acknowledge(m);
            } catch (PulsarClientException e) {
                fail("shouldn't have failed ", e);
            }
        });
        // (4) consumer1 will consumer remaining msgs and consumer2 will ack those messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer1.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                totalReceiveMessages++;
                consumer2.acknowledge(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer2.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                totalReceiveMessages++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // verify total-consumer messages = total-produce messages
        assertEquals(totalProducedMsgs, totalReceiveMessages);
        producer.close();
        consumer1.close();
        consumer2.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 8 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testSharedConsumerAckDifferentConsumer.

/**
 * consume message from consumer1 and send acknowledgement from different consumer subscribed under same
 * subscription-name
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testSharedConsumerAckDifferentConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(1);
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
    Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<Message> consumerMsgSet1 = Sets.newHashSet();
    Set<Message> consumerMsgSet2 = Sets.newHashSet();
    for (int i = 0; i < 5; i++) {
        msg = consumer1.receive();
        consumerMsgSet1.add(msg);
        msg = consumer2.receive();
        consumerMsgSet2.add(msg);
    }
    consumerMsgSet1.stream().forEach(m -> {
        try {
            consumer2.acknowledge(m);
        } catch (PulsarClientException e) {
            fail();
        }
    });
    consumerMsgSet2.stream().forEach(m -> {
        try {
            consumer1.acknowledge(m);
        } catch (PulsarClientException e) {
            fail();
        }
    });
    consumer1.redeliverUnacknowledgedMessages();
    consumer2.redeliverUnacknowledgedMessages();
    try {
        if (consumer1.receive(100, TimeUnit.MILLISECONDS) != null || consumer2.receive(100, TimeUnit.MILLISECONDS) != null) {
            fail();
        }
    } finally {
        consumer1.close();
        consumer2.close();
    }
    log.info("-- Exiting {} test --", methodName);
}

Example 9 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testConsumerBlockingWithUnAckedMessages.

/**
 * Verify: Consumer stops receiving msg when reach unack-msg limit and starts receiving once acks messages 1.
 * Produce X (600) messages 2. Consumer has receive size (10) and receive message without acknowledging 3. Consumer
 * will stop receiving message after unAckThreshold = 500 4. Consumer acks messages and starts consuming remanining
 * messages This testcase enables checksum sending while producing message and broker verifies the checksum for the
 * message.
 *
 * @throws Exception
 */
@Test
public void testConsumerBlockingWithUnAckedMessages() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 500;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 600;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
        assertEquals(messages.size(), unAckedMessagesBufferSize);
        // start acknowledging messages
        messages.forEach(m -> {
            try {
                consumer.acknowledge(m);
            } catch (PulsarClientException e) {
                fail("ack failed", e);
            }
        });
        // try to consume remaining messages
        int remainingMessages = totalProducedMsgs - messages.size();
        for (int i = 0; i < remainingMessages; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            }
        }
        // total received-messages should match to produced messages
        assertEquals(totalProducedMsgs, messages.size());
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 10 with Consumer

use of org.apache.pulsar.client.api.Consumer in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testUnackedBlockAtBatch.

@Test(dataProvider = "batch")
public void testUnackedBlockAtBatch(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int maxUnackedMessages = 20;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 100;
        int totalReceiveMessages = 0;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnackedMessages);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        if (batchMessageDelayMs != 0) {
            producerConf.setBatchingEnabled(true);
            producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
            producerConf.setBatchingMaxMessages(5);
        }
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        List<CompletableFuture<MessageId>> futures = Lists.newArrayList();
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            futures.add(producer.sendAsync(message.getBytes()));
        }
        FutureUtil.waitForAll(futures).get();
        // (2) Consumer1: consume without ack:
        // try to consume messages: but will be able to consume number of messages = maxUnackedMessages
        Message msg = null;
        List<Message> messages = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer1.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                totalReceiveMessages++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // should be blocked due to unack-msgs and should not consume all msgs
        assertNotEquals(messages.size(), totalProducedMsgs);
        // ack for all maxUnackedMessages
        messages.forEach(m -> {
            try {
                consumer1.acknowledge(m);
            } catch (PulsarClientException e) {
                fail("shouldn't have failed ", e);
            }
        });
        // (3) Consumer consumes and ack: so it should consume all remaining messages
        messages.clear();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer1.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                totalReceiveMessages++;
                consumer1.acknowledge(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // verify total-consumer messages = total-produce messages
        assertEquals(totalProducedMsgs, totalReceiveMessages);
        producer.close();
        consumer1.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}