Examples with ConsumerImpl org.apache.pulsar.client.impl.ConsumerImpl used on opensource projects

Example 1 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class PersistentQueueE2ETest method testUnackedCountWithRedeliveries.

@Test
public void testUnackedCountWithRedeliveries() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/testUnackedCountWithRedeliveries";
    final String subName = "sub3";
    final int numMsgs = 10;
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).receiverQueueSize(10).subscriptionType(SubscriptionType.Shared);
    ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
    for (int i = 0; i < numMsgs; i++) {
        producer.send(("hello-" + i).getBytes());
    }
    Set<MessageId> c1_receivedMessages = new HashSet<>();
    // C-1 gets all messages but doesn't ack
    for (int i = 0; i < numMsgs; i++) {
        c1_receivedMessages.add(consumer1.receive().getMessageId());
    }
    // C-2 will not get any message initially, since everything went to C-1 already
    Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
    // Trigger C-1 to redeliver everything, half will go C-1 again and the other half to C-2
    consumer1.redeliverUnacknowledgedMessages(c1_receivedMessages);
    // Consumer 2 will also receive all message but not ack
    for (int i = 0; i < numMsgs; i++) {
        consumer2.receive();
    }
    for (MessageId msgId : c1_receivedMessages) {
        consumer1.acknowledge(msgId);
    }
    PersistentTopicStats stats = admin.persistentTopics().getStats(topicName);
    // Unacked messages count should be 0 for both consumers at this point
    SubscriptionStats subStats = stats.subscriptions.get(subName);
    assertEquals(subStats.msgBacklog, 0);
    for (ConsumerStats cs : subStats.consumers) {
        assertEquals(cs.unackedMessages, 0);
    }
    producer.close();
    consumer1.close();
    consumer2.close();
    admin.persistentTopics().delete(topicName);
}

Example 2 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause.

/**
 * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
 * blocked due to unacked messsages
 *
 * Usecase: produce message with 10ms interval: so, consumer can consume only 10 messages without acking
 *
 * @throws Exception
 */
@Test
public void testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 10;
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 20;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
            Thread.sleep(10);
        }
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages1 = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages1.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client should not receive all produced messages and should be blocked due to unack-messages
        assertEquals(messages1.size(), unAckedMessagesBufferSize);
        Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
            return (MessageIdImpl) m.getMessageId();
        }).collect(Collectors.toSet());
        // (3) redeliver all consumed messages
        consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
        Thread.sleep(1000);
        Set<MessageIdImpl> messages2 = Sets.newHashSet();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages2.add((MessageIdImpl) msg.getMessageId());
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        assertEquals(messages1.size(), messages2.size());
        // (4) Verify: redelivered all previous unacked-consumed messages
        messages2.removeAll(redeliveryMessages);
        assertEquals(messages2.size(), 0);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 3 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testUnackBlockRedeliverMessages.

@Test
public void testUnackBlockRedeliverMessages() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    int totalReceiveMsg = 0;
    try {
        final int unAckedMessagesBufferSize = 20;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 100;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                totalReceiveMsg++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        consumer.redeliverUnacknowledgedMessages();
        Thread.sleep(1000);
        int alreadyConsumedMessages = messages.size();
        messages.clear();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                consumer.acknowledge(msg);
                totalReceiveMsg++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // total received-messages should match to produced messages
        assertEquals(totalProducedMsgs + alreadyConsumedMessages, totalReceiveMsg);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 4 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testRedeliveryFailOverConsumer.

@Test
public void testRedeliveryFailOverConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int receiverQueueSize = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(receiverQueueSize);
    conf.setSubscriptionType(SubscriptionType.Failover);
    // Only subscribe consumer
    ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
    // (1) First round to produce-consume messages
    int consumeMsgInParts = 4;
    for (int i = 0; i < receiverQueueSize; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        Thread.sleep(10);
    }
    // (1.a) consume first consumeMsgInParts msgs and trigger redeliver
    Message msg = null;
    List<Message> messages1 = Lists.newArrayList();
    for (int i = 0; i < consumeMsgInParts; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg != null) {
            messages1.add(msg);
            consumer.acknowledge(msg);
            log.info("Received message: " + new String(msg.getData()));
        } else {
            break;
        }
    }
    assertEquals(messages1.size(), consumeMsgInParts);
    consumer.redeliverUnacknowledgedMessages();
    // (1.b) consume second consumeMsgInParts msgs and trigger redeliver
    messages1.clear();
    for (int i = 0; i < consumeMsgInParts; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg != null) {
            messages1.add(msg);
            consumer.acknowledge(msg);
            log.info("Received message: " + new String(msg.getData()));
        } else {
            break;
        }
    }
    assertEquals(messages1.size(), consumeMsgInParts);
    consumer.redeliverUnacknowledgedMessages();
    // (2) Second round to produce-consume messages
    for (int i = 0; i < receiverQueueSize; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        Thread.sleep(100);
    }
    int remainingMsgs = (2 * receiverQueueSize) - (2 * consumeMsgInParts);
    messages1.clear();
    for (int i = 0; i < remainingMsgs; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg != null) {
            messages1.add(msg);
            consumer.acknowledge(msg);
            log.info("Received message: " + new String(msg.getData()));
        } else {
            break;
        }
    }
    assertEquals(messages1.size(), remainingMsgs);
    producer.close();
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 5 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testBlockUnackedConsumerRedeliverySpecificMessagesCloseConsumerWhileProduce.

/**
 * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
 * blocked due to unacked messsages
 *
 * Usecase: Consumer starts consuming only after all messages have been produced. So, consumer consumes total
 * receiver-queue-size number messages => ask for redelivery and receives all messages again.
 *
 * @throws Exception
 */
@Test
public void testBlockUnackedConsumerRedeliverySpecificMessagesCloseConsumerWhileProduce() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 10;
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 50;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        // Only subscribe consumer
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        consumer.close();
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
            Thread.sleep(10);
        }
        // (1.a) start consumer again
        consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages1 = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages1.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client should not receive all produced messages and should be blocked due to unack-messages
        assertEquals(messages1.size(), receiverQueueSize);
        Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
            return (MessageIdImpl) m.getMessageId();
        }).collect(Collectors.toSet());
        // (3) redeliver all consumed messages
        consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
        Thread.sleep(1000);
        Set<MessageIdImpl> messages2 = Sets.newHashSet();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages2.add((MessageIdImpl) msg.getMessageId());
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        assertEquals(messages1.size(), messages2.size());
        // (4) Verify: redelivered all previous unacked-consumed messages
        messages2.removeAll(redeliveryMessages);
        assertEquals(messages2.size(), 0);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}