Examples with PersistentDispatcherMultipleConsumers org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers used on opensource projects

Example 6 with PersistentDispatcherMultipleConsumers

use of org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers in project incubator-pulsar by apache.

the class BatchMessageTest method testConcurrentBatchMessageAck.

/**
 * Verifies batch-message acking is thread-safe
 *
 * @throws Exception
 */
@Test(timeOut = 3000)
public void testConcurrentBatchMessageAck() throws Exception {
    int numMsgs = 10;
    final String topicName = "persistent://prop/use/ns-abc/testConcurrentAck";
    final String subscriptionName = "sub-1";
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).subscribe();
    consumer.close();
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).batchingMaxPublishDelay(5, TimeUnit.SECONDS).batchingMaxMessages(numMsgs).enableBatching(true).create();
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("my-message-" + i).getBytes();
        Message<byte[]> msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    final Consumer<byte[]> myConsumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).subscribe();
    // assertEquals(dispatcher.getTotalUnackedMessages(), 1);
    ExecutorService executor = Executors.newFixedThreadPool(10);
    final CountDownLatch latch = new CountDownLatch(numMsgs);
    final AtomicBoolean failed = new AtomicBoolean(false);
    for (int i = 0; i < numMsgs; i++) {
        executor.submit(() -> {
            try {
                Message<byte[]> msg = myConsumer.receive(1, TimeUnit.SECONDS);
                myConsumer.acknowledge(msg);
            } catch (Exception e) {
                failed.set(false);
            }
            latch.countDown();
        });
    }
    latch.await();
    PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) topic.getSubscription(subscriptionName).getDispatcher();
    // check strategically to let ack-message receive by broker
    retryStrategically((test) -> dispatcher.getConsumers().get(0).getUnackedMessages() == 0, 5, 150);
    assertEquals(dispatcher.getConsumers().get(0).getUnackedMessages(), 0);
    executor.shutdown();
    myConsumer.close();
    producer.close();
}

Example 7 with PersistentDispatcherMultipleConsumers

use of org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers in project incubator-pulsar by apache.

the class PersistentDispatcherFailoverConsumerTest method testMultipleDispatcherGetNextConsumerWithDifferentPriorityLevel.

@Test
public void testMultipleDispatcherGetNextConsumerWithDifferentPriorityLevel() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock);
    Consumer consumer1 = createConsumer(0, 2, false, 1);
    Consumer consumer2 = createConsumer(0, 2, false, 2);
    Consumer consumer3 = createConsumer(0, 2, false, 3);
    Consumer consumer4 = createConsumer(1, 2, false, 4);
    Consumer consumer5 = createConsumer(1, 1, false, 5);
    Consumer consumer6 = createConsumer(1, 2, false, 6);
    Consumer consumer7 = createConsumer(2, 1, false, 7);
    Consumer consumer8 = createConsumer(2, 1, false, 8);
    Consumer consumer9 = createConsumer(2, 1, false, 9);
    dispatcher.addConsumer(consumer1);
    dispatcher.addConsumer(consumer2);
    dispatcher.addConsumer(consumer3);
    dispatcher.addConsumer(consumer4);
    dispatcher.addConsumer(consumer5);
    dispatcher.addConsumer(consumer6);
    dispatcher.addConsumer(consumer7);
    dispatcher.addConsumer(consumer8);
    dispatcher.addConsumer(consumer9);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer5);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer6);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer6);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer7);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer8);
    // in between add upper priority consumer with more permits
    Consumer consumer10 = createConsumer(0, 2, false, 10);
    dispatcher.addConsumer(consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer9);
}

Example 8 with PersistentDispatcherMultipleConsumers

use of org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers in project incubator-pulsar by apache.

the class PersistentDispatcherFailoverConsumerTest method testFewBlockedConsumerDifferentPriority.

@Test
public void testFewBlockedConsumerDifferentPriority() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock);
    Consumer consumer1 = createConsumer(0, 2, false, 1);
    Consumer consumer2 = createConsumer(0, 2, false, 2);
    Consumer consumer3 = createConsumer(0, 2, false, 3);
    Consumer consumer4 = createConsumer(0, 2, false, 4);
    Consumer consumer5 = createConsumer(0, 1, true, 5);
    Consumer consumer6 = createConsumer(0, 2, true, 6);
    Consumer consumer7 = createConsumer(1, 2, false, 7);
    Consumer consumer8 = createConsumer(1, 10, true, 8);
    Consumer consumer9 = createConsumer(1, 2, false, 9);
    Consumer consumer10 = createConsumer(2, 2, false, 10);
    Consumer consumer11 = createConsumer(2, 10, true, 11);
    Consumer consumer12 = createConsumer(2, 2, false, 12);
    dispatcher.addConsumer(consumer1);
    dispatcher.addConsumer(consumer2);
    dispatcher.addConsumer(consumer3);
    dispatcher.addConsumer(consumer4);
    dispatcher.addConsumer(consumer5);
    dispatcher.addConsumer(consumer6);
    dispatcher.addConsumer(consumer7);
    dispatcher.addConsumer(consumer8);
    dispatcher.addConsumer(consumer9);
    dispatcher.addConsumer(consumer10);
    dispatcher.addConsumer(consumer11);
    dispatcher.addConsumer(consumer12);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer1);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer2);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer3);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer7);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer9);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer7);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer9);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer12);
    // add consumer with lower priority again
    Consumer consumer13 = createConsumer(0, 2, false, 13);
    Consumer consumer14 = createConsumer(0, 2, true, 14);
    dispatcher.addConsumer(consumer13);
    dispatcher.addConsumer(consumer14);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer13);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer13);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer10);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer12);
    Assert.assertEquals(getNextConsumer(dispatcher), null);
}

Example 9 with PersistentDispatcherMultipleConsumers

use of org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers in project incubator-pulsar by apache.

the class PersistentDispatcherFailoverConsumerTest method testFewBlockedConsumerDifferentPriority2.

@Test
public void testFewBlockedConsumerDifferentPriority2() throws Exception {
    PersistentTopic topic = new PersistentTopic(successTopicName, ledgerMock, brokerService);
    PersistentDispatcherMultipleConsumers dispatcher = new PersistentDispatcherMultipleConsumers(topic, cursorMock);
    Consumer consumer1 = createConsumer(0, 2, true, 1);
    Consumer consumer2 = createConsumer(0, 2, true, 2);
    Consumer consumer3 = createConsumer(0, 2, true, 3);
    Consumer consumer4 = createConsumer(1, 2, false, 4);
    Consumer consumer5 = createConsumer(1, 1, false, 5);
    Consumer consumer6 = createConsumer(2, 1, false, 6);
    Consumer consumer7 = createConsumer(2, 2, true, 7);
    dispatcher.addConsumer(consumer1);
    dispatcher.addConsumer(consumer2);
    dispatcher.addConsumer(consumer3);
    dispatcher.addConsumer(consumer4);
    dispatcher.addConsumer(consumer5);
    dispatcher.addConsumer(consumer6);
    dispatcher.addConsumer(consumer7);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer5);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer4);
    Assert.assertEquals(getNextConsumer(dispatcher), consumer6);
    Assert.assertEquals(getNextConsumer(dispatcher), null);
}

Example 10 with PersistentDispatcherMultipleConsumers

use of org.apache.pulsar.broker.service.persistent.PersistentDispatcherMultipleConsumers in project incubator-pulsar by apache.

the class BrokerService method blockDispatchersWithLargeUnAckMessages.

private void blockDispatchersWithLargeUnAckMessages() {
    lock.readLock().lock();
    try {
        topics.forEach((name, topicFuture) -> {
            if (topicFuture.isDone()) {
                try {
                    topicFuture.get().getSubscriptions().forEach((subName, persistentSubscription) -> {
                        if (persistentSubscription.getDispatcher() instanceof PersistentDispatcherMultipleConsumers) {
                            PersistentDispatcherMultipleConsumers dispatcher = (PersistentDispatcherMultipleConsumers) persistentSubscription.getDispatcher();
                            int dispatcherUnAckMsgs = dispatcher.getTotalUnackedMessages();
                            if (dispatcherUnAckMsgs > maxUnackedMsgsPerDispatcher) {
                                log.info("[{}] Blocking dispatcher due to reached max broker limit {}", dispatcher.getName(), dispatcher.getTotalUnackedMessages());
                                dispatcher.blockDispatcherOnUnackedMsgs();
                                blockedDispatchers.add(dispatcher);
                            }
                        }
                    });
                } catch (Exception e) {
                    log.warn("Failed to get topic from future ", e);
                }
            }
        });
    } finally {
        lock.readLock().unlock();
    }
}