Examples with ConsumerImpl org.apache.pulsar.client.impl.ConsumerImpl used on opensource projects

Example 6 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class DispatcherBlockConsumerTest method testBrokerDispatchBlockAndSubAckBackRequiredMsgs.

/**
 * Verifies if broker is already blocked multiple subscriptions if one of them acked back perBrokerDispatcherLimit
 * messages then that dispatcher gets unblocked and starts consuming messages
 *
 * <pre>
 * 1. subscription-1 consume messages and doesn't ack so it reaches maxUnAckPerBroker(200) and blocks sub-1
 * 2. subscription-2 can consume only dispatcherLimitWhenBrokerIsBlocked(20) and then sub-2 gets blocked
 * 3. subscription-2 acks back 10 messages (dispatcherLimitWhenBrokerIsBlocked/2) to gets unblock
 * 4. sub-2 starts acking once it gets unblocked and it consumes all published messages
 * </pre>
 */
@SuppressWarnings("unchecked")
@Test
public void testBrokerDispatchBlockAndSubAckBackRequiredMsgs() {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerBroker();
    double unAckedMessagePercentage = pulsar.getConfiguration().getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked();
    try {
        final int maxUnAckPerBroker = 200;
        final double unAckMsgPercentagePerDispatcher = 10;
        // 200 *
        int maxUnAckPerDispatcher = (int) ((maxUnAckPerBroker * unAckMsgPercentagePerDispatcher) / 100);
        // 10% = 20
        // messages
        pulsar.getConfiguration().setMaxUnackedMessagesPerBroker(maxUnAckPerBroker);
        pulsar.getConfiguration().setMaxUnackedMessagesPerSubscriptionOnBrokerBlocked(unAckMsgPercentagePerDispatcher);
        stopBroker();
        startBroker();
        Field field = BrokerService.class.getDeclaredField("blockedDispatchers");
        field.setAccessible(true);
        ConcurrentOpenHashSet<PersistentDispatcherMultipleConsumers> blockedDispatchers = (ConcurrentOpenHashSet<PersistentDispatcherMultipleConsumers>) field.get(pulsar.getBrokerService());
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = maxUnAckPerBroker * 3;
        final String topicName = "persistent://my-property/use/my-ns/unacked-topic";
        final String subscriberName1 = "subscriber-1";
        final String subscriberName2 = "subscriber-2";
        ConsumerImpl<byte[]> consumer1Sub1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName1).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
        // create subscription-2 and 3
        ConsumerImpl<byte[]> consumer1Sub2 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName2).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
        consumer1Sub2.close();
        // continuously checks unack-message dispatching
        ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
        executor.scheduleAtFixedRate(() -> pulsar.getBrokerService().checkUnAckMessageDispatching(), 10, 10, TimeUnit.MILLISECONDS);
        Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/unacked-topic").create();
        // Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        /**
         ***
         * (1) try to consume messages: without acking messages and dispatcher will be blocked once it reaches
         * maxUnAckPerBroker limit
         **
         */
        Message<?> msg = null;
        Set<MessageId> messages1 = Sets.newHashSet();
        for (int j = 0; j < totalProducedMsgs; j++) {
            msg = consumer1Sub1.receive(100, TimeUnit.MILLISECONDS);
            if (msg != null) {
                messages1.add(msg.getMessageId());
            } else {
                break;
            }
            // subscription
            if (j == maxUnAckPerBroker) {
                Thread.sleep(200);
            }
        }
        // client must receive number of messages = maxUnAckPerbroker rather all produced messages
        assertNotEquals(messages1.size(), totalProducedMsgs);
        // (1.b) consumer2 with same sub should not receive any more messages as subscription is blocked
        ConsumerImpl<byte[]> consumer2Sub1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName1).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
        int consumer2Msgs = 0;
        for (int j = 0; j < totalProducedMsgs; j++) {
            msg = consumer2Sub1.receive(100, TimeUnit.MILLISECONDS);
            if (msg != null) {
                consumer2Msgs++;
            } else {
                break;
            }
        }
        // consumer should not consume any more messages as broker has blocked the dispatcher
        assertEquals(consumer2Msgs, 0);
        consumer2Sub1.close();
        // (1.c) verify that dispatcher is part of blocked dispatcher
        assertEquals(blockedDispatchers.size(), 1);
        String dispatcherName = blockedDispatchers.values().get(0).getName();
        String subName = dispatcherName.substring(dispatcherName.lastIndexOf("/") + 2, dispatcherName.length());
        assertEquals(subName, subscriberName1);
        /**
         * (2) However, other subscription2 should still be able to consume messages until it reaches to
         * maxUnAckPerDispatcher limit
         */
        consumer1Sub2 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName2).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
        Set<MessageId> messages2 = Sets.newHashSet();
        for (int j = 0; j < totalProducedMsgs; j++) {
            msg = consumer1Sub2.receive(100, TimeUnit.MILLISECONDS);
            if (msg != null) {
                messages2.add(msg.getMessageId());
            } else {
                break;
            }
        }
        // (2.b) It should receive only messages with limit of maxUnackPerDispatcher
        assertEquals(messages2.size(), maxUnAckPerDispatcher, receiverQueueSize);
        assertEquals(blockedDispatchers.size(), 2);
        // (2.c) Now subscriber-2 is blocked: so acking back should unblock dispatcher
        Iterator<MessageId> itrMsgs = messages2.iterator();
        // eg. 25 -20 = 5
        int additionalMsgConsumedAfterBlocked = messages2.size() - maxUnAckPerDispatcher + 1;
        for (int i = 0; i < (additionalMsgConsumedAfterBlocked + (maxUnAckPerDispatcher / 2)); i++) {
            consumer1Sub2.acknowledge(itrMsgs.next());
        }
        // let ack completed
        Thread.sleep(1000);
        // verify subscriber2 is unblocked and ready to consume more messages
        assertEquals(blockedDispatchers.size(), 1);
        for (int j = 0; j < totalProducedMsgs; j++) {
            msg = consumer1Sub2.receive(200, TimeUnit.MILLISECONDS);
            if (msg != null) {
                messages2.add(msg.getMessageId());
                consumer1Sub2.acknowledge(msg);
            } else {
                break;
            }
        }
        // verify it consumed all messages now
        assertEquals(messages2.size(), totalProducedMsgs);
        consumer1Sub1.close();
        consumer1Sub2.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerBroker(unAckedMessages);
        pulsar.getConfiguration().setMaxUnackedMessagesPerSubscriptionOnBrokerBlocked(unAckedMessagePercentage);
    }
}

Example 7 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class DispatcherBlockConsumerTest method testRedeliveryOnBlockedDistpatcher.

/**
 * Verifies: old-client which does redelivery of all messages makes broker to redeliver all unacked messages for
 * redelivery.
 *
 * @throws Exception
 */
@Test(timeOut = 10000)
public void testRedeliveryOnBlockedDistpatcher() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerSubscription();
    try {
        stopBroker();
        startBroker();
        final int unackMsgAllowed = 100;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 200;
        final String topicName = "persistent://my-property/use/my-ns/unacked-topic";
        final String subscriberName = "subscriber-1";
        pulsar.getConfiguration().setMaxUnackedMessagesPerSubscription(unackMsgAllowed);
        ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared);
        ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
        ConsumerImpl<byte[]> consumer2 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
        ConsumerImpl<byte[]> consumer3 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
        List<ConsumerImpl<?>> consumers = Lists.newArrayList(consumer1, consumer2, consumer3);
        Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/unacked-topic").create();
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) try to consume messages: but will be able to consume number of messages = unackMsgAllowed
        Message<?> msg = null;
        Set<MessageId> messages = Sets.newHashSet();
        for (int i = 0; i < 3; i++) {
            for (int j = 0; j < totalProducedMsgs; j++) {
                msg = consumers.get(i).receive(500, TimeUnit.MILLISECONDS);
                if (msg != null) {
                    messages.add(msg.getMessageId());
                    log.info("Received message: " + new String(msg.getData()));
                } else {
                    break;
                }
            }
        }
        int totalConsumedMsgs = messages.size();
        // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
        assertEquals(totalConsumedMsgs, unackMsgAllowed, 3 * receiverQueueSize);
        // trigger redelivery
        consumers.forEach(c -> {
            c.redeliverUnacknowledgedMessages();
        });
        // wait for redelivery to be completed
        Thread.sleep(1000);
        // now, broker must have redelivered all unacked messages
        Map<ConsumerImpl<?>, Set<MessageId>> messages1 = Maps.newHashMap();
        for (int i = 0; i < 3; i++) {
            for (int j = 0; j < totalProducedMsgs; j++) {
                msg = consumers.get(i).receive(500, TimeUnit.MILLISECONDS);
                if (msg != null) {
                    messages1.putIfAbsent(consumers.get(i), Sets.newHashSet());
                    messages1.get(consumers.get(i)).add(msg.getMessageId());
                    log.info("Received message: " + new String(msg.getData()));
                } else {
                    break;
                }
            }
        }
        Set<MessageId> result = Sets.newHashSet();
        messages1.values().forEach(result::addAll);
        // check all unacked messages have been redelivered
        assertEquals(totalConsumedMsgs, result.size(), 3 * receiverQueueSize);
        // start acknowledging messages
        messages1.forEach((c, msgs) -> {
            msgs.forEach(m -> {
                try {
                    c.acknowledge(m);
                } catch (PulsarClientException e) {
                    fail("ack failed", e);
                }
            });
        });
        messages1.values().forEach(result::addAll);
        // try to consume remaining messages
        int remainingMessages = totalProducedMsgs - result.size();
        // try to consume remaining messages: broker may take time to deliver so, retry multiple time to consume
        // all messages
        CountDownLatch latch = new CountDownLatch(remainingMessages);
        Queue<MessageId> consumedMessages = Queues.newConcurrentLinkedQueue();
        for (int i = 0; i < consumers.size(); i++) {
            final int counsumerIndex = i;
            for (int j = 0; j < remainingMessages; j++) {
                consumers.get(i).receiveAsync().thenAccept(m -> {
                    consumedMessages.add(m.getMessageId());
                    try {
                        consumers.get(counsumerIndex).acknowledge(m);
                    } catch (PulsarClientException e) {
                        fail("failed to ack", e);
                    }
                    latch.countDown();
                });
            }
        }
        latch.await();
        // total received-messages should match remaining messages excluding duplicate
        assertTrue(consumedMessages.size() >= remainingMessages);
        producer.close();
        consumers.forEach(c -> {
            try {
                c.close();
            } catch (PulsarClientException e) {
            }
        });
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 8 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class DispatcherBlockConsumerTest method testBlockBrokerDispatching.

/**
 * </pre>
 * verifies perBroker dispatching blocking. A. maxUnAckPerBroker = 200, maxUnAckPerDispatcher = 20 Now, it tests
 * with 3 subscriptions.
 *
 * 1. Subscription-1: try to consume without acking a. consumer will be blocked after 200 (maxUnAckPerBroker) msgs
 * b. even second consumer will not receive any new messages c. broker will have 1 blocked dispatcher 2.
 * Subscription-2: try to consume without acking a. as broker is already blocked it will block subscription after 20
 * msgs (maxUnAckPerDispatcher) b. broker will have 2 blocked dispatchers 3. Subscription-3: try to consume with
 * acking a. as consumer is acking not reached maxUnAckPerDispatcher=20 unack msg => consumes all produced msgs
 * 4.Subscription-1 : acks all pending msgs and consume by acking a. broker unblocks all dispatcher and sub-1
 * consumes all messages 5. Subscription-2 : it triggers redelivery and acks all messages so, it consumes all
 * produced messages
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 10000)
public void testBlockBrokerDispatching() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerBroker();
    double unAckedMessagePercentage = pulsar.getConfiguration().getMaxUnackedMessagesPerSubscriptionOnBrokerBlocked();
    try {
        final int maxUnAckPerBroker = 200;
        final double unAckMsgPercentagePerDispatcher = 10;
        // 200 *
        int maxUnAckPerDispatcher = (int) ((maxUnAckPerBroker * unAckMsgPercentagePerDispatcher) / 100);
        // 10% = 20
        // messages
        pulsar.getConfiguration().setMaxUnackedMessagesPerBroker(maxUnAckPerBroker);
        pulsar.getConfiguration().setMaxUnackedMessagesPerSubscriptionOnBrokerBlocked(unAckMsgPercentagePerDispatcher);
        stopBroker();
        startBroker();
        Field field = BrokerService.class.getDeclaredField("blockedDispatchers");
        field.setAccessible(true);
        @SuppressWarnings("unchecked") ConcurrentOpenHashSet<PersistentDispatcherMultipleConsumers> blockedDispatchers = (ConcurrentOpenHashSet<PersistentDispatcherMultipleConsumers>) field.get(pulsar.getBrokerService());
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = maxUnAckPerBroker * 3;
        final String topicName = "persistent://my-property/use/my-ns/unacked-topic";
        final String subscriberName1 = "subscriber-1";
        final String subscriberName2 = "subscriber-2";
        final String subscriberName3 = "subscriber-3";
        ConsumerImpl<byte[]> consumer1Sub1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName1).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
        // create subscription-2 and 3
        ConsumerImpl<byte[]> consumer1Sub2 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName2).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
        consumer1Sub2.close();
        ConsumerImpl<byte[]> consumer1Sub3 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName3).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
        consumer1Sub3.close();
        Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/unacked-topic").create();
        // continuously checks unack-message dispatching
        ScheduledExecutorService executor = Executors.newScheduledThreadPool(1);
        executor.scheduleAtFixedRate(() -> pulsar.getBrokerService().checkUnAckMessageDispatching(), 10, 10, TimeUnit.MILLISECONDS);
        // Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        /**
         ***
         * (1) try to consume messages: without acking messages and dispatcher will be blocked once it reaches
         * maxUnAckPerBroker limit
         **
         */
        Message<byte[]> msg = null;
        Set<MessageId> messages1 = Sets.newHashSet();
        for (int j = 0; j < totalProducedMsgs; j++) {
            msg = consumer1Sub1.receive(100, TimeUnit.MILLISECONDS);
            if (msg != null) {
                messages1.add(msg.getMessageId());
            } else {
                break;
            }
            // subscription
            if (j == maxUnAckPerBroker) {
                Thread.sleep(200);
            }
        }
        // client must receive number of messages = maxUnAckPerbroker rather all produced messages
        assertNotEquals(messages1.size(), totalProducedMsgs);
        // (1.b) consumer2 with same sub should not receive any more messages as subscription is blocked
        ConsumerImpl<byte[]> consumer2Sub1 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName1).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
        int consumer2Msgs = 0;
        for (int j = 0; j < totalProducedMsgs; j++) {
            msg = consumer2Sub1.receive(100, TimeUnit.MILLISECONDS);
            if (msg != null) {
                consumer2Msgs++;
            } else {
                break;
            }
        }
        // consumer should not consume any more messages as broker has blocked the dispatcher
        assertEquals(consumer2Msgs, 0);
        consumer2Sub1.close();
        // (1.c) verify that dispatcher is part of blocked dispatcher
        assertEquals(blockedDispatchers.size(), 1);
        String dispatcherName = blockedDispatchers.values().get(0).getName();
        String subName = dispatcherName.substring(dispatcherName.lastIndexOf("/") + 2, dispatcherName.length());
        assertEquals(subName, subscriberName1);
        /**
         * (2) However, other subscription2 should still be able to consume messages until it reaches to
         * maxUnAckPerDispatcher limit
         */
        ConsumerImpl<byte[]> consumerSub2 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName2).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
        Set<MessageId> messages2 = Sets.newHashSet();
        for (int j = 0; j < totalProducedMsgs; j++) {
            msg = consumerSub2.receive(100, TimeUnit.MILLISECONDS);
            if (msg != null) {
                messages2.add(msg.getMessageId());
            } else {
                break;
            }
        }
        // (2.b) It should receive only messages with limit of maxUnackPerDispatcher
        assertEquals(messages2.size(), maxUnAckPerDispatcher, receiverQueueSize);
        assertEquals(blockedDispatchers.size(), 2);
        /**
         * (3) if Subscription3 is acking then it shouldn't be blocked *
         */
        consumer1Sub3 = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName3).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared).subscribe();
        int consumedMsgsSub3 = 0;
        for (int j = 0; j < totalProducedMsgs; j++) {
            msg = consumer1Sub3.receive(100, TimeUnit.MILLISECONDS);
            if (msg != null) {
                consumedMsgsSub3++;
                consumer1Sub3.acknowledge(msg);
            } else {
                break;
            }
        }
        assertEquals(consumedMsgsSub3, totalProducedMsgs);
        assertEquals(blockedDispatchers.size(), 2);
        /**
         * (4) try to ack messages from sub1 which should unblock broker
         */
        messages1.forEach(consumer1Sub1::acknowledgeAsync);
        // sleep so, broker receives all ack back to unblock subscription
        Thread.sleep(1000);
        for (int j = 0; j < totalProducedMsgs; j++) {
            msg = consumer1Sub1.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages1.add(msg.getMessageId());
                consumer1Sub1.acknowledge(msg);
            } else {
                break;
            }
        }
        assertEquals(messages1.size(), totalProducedMsgs);
        // it unblocks all consumers
        assertEquals(blockedDispatchers.size(), 0);
        /**
         * (5) try redelivery on sub2 consumer and verify to consume all messages
         */
        consumerSub2.redeliverUnacknowledgedMessages();
        AtomicInteger msgReceivedCount = new AtomicInteger(0);
        CountDownLatch latch = new CountDownLatch(totalProducedMsgs);
        for (int j = 0; j < totalProducedMsgs; j++) {
            consumerSub2.receiveAsync().thenAccept(m -> {
                msgReceivedCount.incrementAndGet();
                latch.countDown();
                try {
                    consumerSub2.acknowledge(m);
                } catch (PulsarClientException e) {
                    fail("failed to ack msg", e);
                }
            });
        }
        latch.await();
        assertEquals(msgReceivedCount.get(), totalProducedMsgs);
        consumer1Sub1.close();
        consumerSub2.close();
        consumer1Sub3.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerBroker(unAckedMessages);
        pulsar.getConfiguration().setMaxUnackedMessagesPerSubscriptionOnBrokerBlocked(unAckedMessagePercentage);
    }
}

Example 9 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class SimpleProducerConsumerTest method testConcurrentConsumerReceiveWhileReconnect.

// This is to test that the flow control counter doesn't get corrupted while concurrent receives during
// reconnections
@Test(dataProvider = "batch")
public void testConcurrentConsumerReceiveWhileReconnect(int batchMessageDelayMs) throws Exception {
    final int recvQueueSize = 100;
    final int numConsumersThreads = 10;
    String subName = UUID.randomUUID().toString();
    final Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://my-property/use/my-ns/my-topic7").subscriptionName(subName).receiverQueueSize(recvQueueSize).subscribe();
    ExecutorService executor = Executors.newCachedThreadPool();
    final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    // there will be 10 threads calling receive() from the same consumer and will block
    Thread.sleep(100);
    // we restart the broker to reconnect
    restartBroker();
    Thread.sleep(2000);
    // publish 100 messages so that the consumers blocked on receive() will now get the messages
    ProducerBuilder<byte[]> producerBuilder = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/my-topic7");
    if (batchMessageDelayMs != 0) {
        producerBuilder.batchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerBuilder.batchingMaxMessages(5);
        producerBuilder.enableBatching(true);
    }
    Producer<byte[]> producer = producerBuilder.create();
    for (int i = 0; i < recvQueueSize; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(500);
    ConsumerImpl<byte[]> consumerImpl = (ConsumerImpl<byte[]>) consumer;
    // The available permits should be 10 and num messages in the queue should be 90
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
    barrier.reset();
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    Thread.sleep(100);
    // The available permits should be 20 and num messages in the queue should be 80
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads * 2);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - (numConsumersThreads * 2));
    // clear the queue
    while (true) {
        Message<byte[]> msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
    }
    // The available permits should be 0 and num messages in the queue should be 0
    Assert.assertEquals(consumerImpl.getAvailablePermits(), 0);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), 0);
    barrier.reset();
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    // we again make 10 threads call receive() and get blocked
    Thread.sleep(100);
    restartBroker();
    Thread.sleep(2000);
    // The available permits should be 10 and num messages in the queue should be 90
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
    consumer.close();
}

Example 10 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testShouldNotBlockConsumerIfRedeliverBeforeReceive.

@Test
public void testShouldNotBlockConsumerIfRedeliverBeforeReceive() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    int totalReceiveMsg = 0;
    try {
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 100;
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setAckTimeout(1, TimeUnit.SECONDS);
        conf.setSubscriptionType(SubscriptionType.Shared);
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) wait for consumer to receive messages
        Thread.sleep(1000);
        assertEquals(consumer.numMessagesInQueue(), receiverQueueSize);
        // (3) wait for messages to expire, we should've received more
        Thread.sleep(2000);
        assertEquals(consumer.numMessagesInQueue(), receiverQueueSize);
        for (int i = 0; i < totalProducedMsgs; i++) {
            Message msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                consumer.acknowledge(msg);
                totalReceiveMsg++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // total received-messages should match to produced messages
        assertEquals(totalProducedMsgs, totalReceiveMsg);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}