Examples with ConsumerImpl com.yahoo.pulsar.client.impl.ConsumerImpl used on opensource projects

Example 1 with ConsumerImpl

use of com.yahoo.pulsar.client.impl.ConsumerImpl in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testRedeliveryFailOverConsumer.

@Test
public void testRedeliveryFailOverConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int receiverQueueSize = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(receiverQueueSize);
    conf.setSubscriptionType(SubscriptionType.Failover);
    // Only subscribe consumer
    ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
    // (1) First round to produce-consume messages
    int consumeMsgInParts = 4;
    for (int i = 0; i < receiverQueueSize; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        Thread.sleep(10);
    }
    // (1.a) consume first consumeMsgInParts msgs and trigger redeliver
    Message msg = null;
    List<Message> messages1 = Lists.newArrayList();
    for (int i = 0; i < consumeMsgInParts; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg != null) {
            messages1.add(msg);
            consumer.acknowledge(msg);
            log.info("Received message: " + new String(msg.getData()));
        } else {
            break;
        }
    }
    assertEquals(messages1.size(), consumeMsgInParts);
    consumer.redeliverUnacknowledgedMessages();
    // (1.b) consume second consumeMsgInParts msgs and trigger redeliver
    messages1.clear();
    for (int i = 0; i < consumeMsgInParts; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg != null) {
            messages1.add(msg);
            consumer.acknowledge(msg);
            log.info("Received message: " + new String(msg.getData()));
        } else {
            break;
        }
    }
    assertEquals(messages1.size(), consumeMsgInParts);
    consumer.redeliverUnacknowledgedMessages();
    // (2) Second round to produce-consume messages
    for (int i = 0; i < receiverQueueSize; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        Thread.sleep(100);
    }
    int remainingMsgs = (2 * receiverQueueSize) - (2 * consumeMsgInParts);
    messages1.clear();
    for (int i = 0; i < remainingMsgs; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg != null) {
            messages1.add(msg);
            consumer.acknowledge(msg);
            log.info("Received message: " + new String(msg.getData()));
        } else {
            break;
        }
    }
    assertEquals(messages1.size(), remainingMsgs);
    producer.close();
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 2 with ConsumerImpl

use of com.yahoo.pulsar.client.impl.ConsumerImpl in project pulsar by yahoo.

the class ZeroQueueSizeTest method zeroQueueSizeFailoverSubscription.

@Test()
public void zeroQueueSizeFailoverSubscription() throws PulsarClientException {
    String key = "zeroQueueSizeFailoverSubscription";
    // 1. Config
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    ConsumerConfiguration configuration = new ConsumerConfiguration();
    configuration.setReceiverQueueSize(0);
    configuration.setSubscriptionType(SubscriptionType.Failover);
    configuration.setConsumerName("consumer-1");
    ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
    configuration.setConsumerName("consumer-2");
    ConsumerImpl consumer2 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
    // 4. Produce Messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
    }
    // 5. Consume messages
    Message message;
    for (int i = 0; i < totalMessages / 2; i++) {
        assertEquals(consumer1.numMessagesInQueue(), 0);
        message = consumer1.receive();
        assertEquals(new String(message.getData()), messagePredicate + i);
        assertEquals(consumer1.numMessagesInQueue(), 0);
        log.info("Consumer received : " + new String(message.getData()));
    }
    // 6. Trigger redelivery
    consumer1.redeliverUnacknowledgedMessages();
    // 7. Trigger Failover
    consumer1.close();
    // 8. Receive messages on failed over consumer
    for (int i = 0; i < totalMessages / 2; i++) {
        assertEquals(consumer2.numMessagesInQueue(), 0);
        message = consumer2.receive();
        assertEquals(new String(message.getData()), messagePredicate + i);
        assertEquals(consumer2.numMessagesInQueue(), 0);
        log.info("Consumer received : " + new String(message.getData()));
    }
}

Example 3 with ConsumerImpl

use of com.yahoo.pulsar.client.impl.ConsumerImpl in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testConcurrentConsumerReceiveWhileReconnect.

// This is to test that the flow control counter doesn't get corrupted while concurrent receives during
// reconnections
@Test(dataProvider = "batch")
public void testConcurrentConsumerReceiveWhileReconnect(int batchMessageDelayMs) throws Exception {
    final int recvQueueSize = 100;
    final int numConsumersThreads = 10;
    final ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(recvQueueSize);
    String subName = UUID.randomUUID().toString();
    final Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic7", subName, conf);
    ExecutorService executor = Executors.newCachedThreadPool();
    final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    // there will be 10 threads calling receive() from the same consumer and will block
    Thread.sleep(100);
    // we restart the broker to reconnect
    restartBroker();
    Thread.sleep(2000);
    // publish 100 messages so that the consumers blocked on receive() will now get the messages
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic7", producerConf);
    for (int i = 0; i < recvQueueSize; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(500);
    ConsumerImpl consumerImpl = (ConsumerImpl) consumer;
    // The available permits should be 10 and num messages in the queue should be 90
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
    barrier.reset();
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    Thread.sleep(100);
    // The available permits should be 20 and num messages in the queue should be 80
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads * 2);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - (numConsumersThreads * 2));
    // clear the queue
    while (true) {
        Message msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
    }
    // The available permits should be 0 and num messages in the queue should be 0
    Assert.assertEquals(consumerImpl.getAvailablePermits(), 0);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), 0);
    barrier.reset();
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    // we again make 10 threads call receive() and get blocked
    Thread.sleep(100);
    restartBroker();
    Thread.sleep(2000);
    // The available permits should be 10 and num messages in the queue should be 90
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
    consumer.close();
}

Example 4 with ConsumerImpl

use of com.yahoo.pulsar.client.impl.ConsumerImpl in project pulsar by yahoo.

the class SimpleProducerConsumerTest method testBlockUnackedConsumerRedeliverySpecificMessagesCloseConsumerWhileProduce.

/**
     * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
     * blocked due to unacked messsages
     *
     * Usecase: Consumer starts consuming only after all messages have been produced.
     * So, consumer consumes total receiver-queue-size number messages => ask for redelivery and receives all messages again.
     *
     * @throws Exception
     */
@Test
public void testBlockUnackedConsumerRedeliverySpecificMessagesCloseConsumerWhileProduce() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 10;
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 50;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        // Only subscribe consumer
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        consumer.close();
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
            Thread.sleep(10);
        }
        // (1.a) start consumer again
        consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages1 = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages1.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client should not receive all produced messages and should be blocked due to unack-messages
        assertEquals(messages1.size(), receiverQueueSize);
        Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
            return (MessageIdImpl) m.getMessageId();
        }).collect(Collectors.toSet());
        // (3) redeliver all consumed messages
        consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
        Thread.sleep(1000);
        Set<MessageIdImpl> messages2 = Sets.newHashSet();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages2.add((MessageIdImpl) msg.getMessageId());
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        assertEquals(messages1.size(), messages2.size());
        // (4) Verify: redelivered all previous unacked-consumed messages
        messages2.removeAll(redeliveryMessages);
        assertEquals(messages2.size(), 0);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 5 with ConsumerImpl

use of com.yahoo.pulsar.client.impl.ConsumerImpl in project pulsar by yahoo.

the class ZeroQueueSizeTest method zeroQueueSizeNormalConsumer.

@Test()
public void zeroQueueSizeNormalConsumer() throws PulsarClientException {
    String key = "nonZeroQueueSizeNormalConsumer";
    // 1. Config
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    ConsumerConfiguration configuration = new ConsumerConfiguration();
    configuration.setReceiverQueueSize(0);
    ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Producer produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message
    Message message;
    for (int i = 0; i < totalMessages; i++) {
        assertEquals(consumer.numMessagesInQueue(), 0);
        message = consumer.receive();
        assertEquals(new String(message.getData()), messagePredicate + i);
        assertEquals(consumer.numMessagesInQueue(), 0);
        log.info("Consumer received : " + new String(message.getData()));
    }
}