Examples with ConsumerImpl org.apache.pulsar.client.impl.ConsumerImpl used on opensource projects

Example 11 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testConcurrentConsumerReceiveWhileReconnect.

// This is to test that the flow control counter doesn't get corrupted while concurrent receives during
// reconnections
@Test(dataProvider = "batch")
public void testConcurrentConsumerReceiveWhileReconnect(int batchMessageDelayMs) throws Exception {
    final int recvQueueSize = 100;
    final int numConsumersThreads = 10;
    final ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(recvQueueSize);
    String subName = UUID.randomUUID().toString();
    final Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic7", subName, conf);
    ExecutorService executor = Executors.newCachedThreadPool();
    final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    // there will be 10 threads calling receive() from the same consumer and will block
    Thread.sleep(100);
    // we restart the broker to reconnect
    restartBroker();
    Thread.sleep(2000);
    // publish 100 messages so that the consumers blocked on receive() will now get the messages
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic7", producerConf);
    for (int i = 0; i < recvQueueSize; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(500);
    ConsumerImpl consumerImpl = (ConsumerImpl) consumer;
    // The available permits should be 10 and num messages in the queue should be 90
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
    barrier.reset();
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    Thread.sleep(100);
    // The available permits should be 20 and num messages in the queue should be 80
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads * 2);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - (numConsumersThreads * 2));
    // clear the queue
    while (true) {
        Message msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
    }
    // The available permits should be 0 and num messages in the queue should be 0
    Assert.assertEquals(consumerImpl.getAvailablePermits(), 0);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), 0);
    barrier.reset();
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    // we again make 10 threads call receive() and get blocked
    Thread.sleep(100);
    restartBroker();
    Thread.sleep(2000);
    // The available permits should be 10 and num messages in the queue should be 90
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
    consumer.close();
}

Example 12 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class BrokerServiceThrottlingTest method testLookupThrottlingForClientByBrokerInternalRetry.

/**
 * This testcase make sure that once consumer lost connection with broker, it always reconnects with broker by
 * retrying on throttling-error exception also.
 *
 * <pre>
 * 1. all consumers get connected
 * 2. broker restarts with maxConcurrentLookupRequest = 1
 * 3. consumers reconnect and some get TooManyRequestException and again retries
 * 4. eventually all consumers will successfully connect to broker
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testLookupThrottlingForClientByBrokerInternalRetry() throws Exception {
    final String topicName = "persistent://prop/usw/my-ns/newTopic";
    String lookupUrl = new URI("pulsar://localhost:" + BROKER_PORT).toString();
    PulsarClient pulsarClient = PulsarClient.builder().serviceUrl(lookupUrl).statsInterval(0, TimeUnit.SECONDS).ioThreads(20).connectionsPerBroker(20).build();
    upsertLookupPermits(100);
    List<Consumer<byte[]>> consumers = Collections.synchronizedList(Lists.newArrayList());
    ExecutorService executor = Executors.newFixedThreadPool(10);
    final int totalConsumers = 8;
    CountDownLatch latch = new CountDownLatch(totalConsumers);
    for (int i = 0; i < totalConsumers; i++) {
        executor.execute(() -> {
            try {
                consumers.add(pulsarClient.newConsumer().topic(topicName).subscriptionName("mysub").subscriptionType(SubscriptionType.Shared).subscribe());
            } catch (PulsarClientException.TooManyRequestsException e) {
            // ok
            } catch (Exception e) {
                fail("it shouldn't failed");
            }
            latch.countDown();
        });
    }
    latch.await();
    stopBroker();
    conf.setMaxConcurrentLookupRequest(1);
    startBroker();
    // wait strategically for all consumers to reconnect
    retryStrategically((test) -> areAllConsumersConnected(consumers), 5, 500);
    int totalConnectedConsumers = 0;
    for (int i = 0; i < consumers.size(); i++) {
        if (((ConsumerImpl<?>) consumers.get(i)).isConnected()) {
            totalConnectedConsumers++;
        }
        consumers.get(i).close();
    }
    assertEquals(totalConnectedConsumers, totalConsumers);
    executor.shutdown();
    pulsarClient.close();
}

Example 13 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class DispatcherBlockConsumerTest method testConsumerBlockingWithUnAckedMessagesAndRedelivery.

/**
 * Verifies: broker blocks dispatching once unack-msg reaches to max-limit. However, on redelivery it redelivers
 * those already delivered-unacked messages again
 *
 * @throws Exception
 */
@SuppressWarnings("unchecked")
@Test(timeOut = 10000)
public void testConsumerBlockingWithUnAckedMessagesAndRedelivery() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerSubscription();
    try {
        final int unackMsgAllowed = 100;
        final int totalProducedMsgs = 200;
        final int receiverQueueSize = 10;
        final String topicName = "persistent://my-property/use/my-ns/unacked-topic";
        final String subscriberName = "subscriber-1";
        pulsar.getConfiguration().setMaxUnackedMessagesPerSubscription(unackMsgAllowed);
        ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriberName).receiverQueueSize(receiverQueueSize).subscriptionType(SubscriptionType.Shared);
        ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
        ConsumerImpl<byte[]> consumer2 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
        ConsumerImpl<byte[]> consumer3 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
        List<ConsumerImpl<byte[]>> consumers = Lists.newArrayList(consumer1, consumer2, consumer3);
        Producer<byte[]> producer = pulsarClient.newProducer().topic("persistent://my-property/use/my-ns/unacked-topic").create();
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) try to consume messages: but will be able to consume number of messages = unackMsgAllowed
        Message<?> msg = null;
        Multimap<ConsumerImpl<?>, MessageId> messages = ArrayListMultimap.create();
        for (int i = 0; i < 3; i++) {
            for (int j = 0; j < totalProducedMsgs; j++) {
                msg = consumers.get(i).receive(500, TimeUnit.MILLISECONDS);
                if (msg != null) {
                    messages.put(consumers.get(i), msg.getMessageId());
                    log.info("Received message: " + new String(msg.getData()));
                } else {
                    break;
                }
            }
        }
        // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
        assertNotEquals(messages.size(), totalProducedMsgs);
        // (3) trigger redelivery
        messages.asMap().forEach((c, msgs) -> {
            c.redeliverUnacknowledgedMessages(msgs.stream().map(m -> (MessageIdImpl) m).collect(Collectors.toSet()));
        });
        // (4) try to consume remaining messages: broker may take time to deliver so, retry multiple time to consume
        // all messages
        Queue<MessageId> result = Queues.newConcurrentLinkedQueue();
        CountDownLatch latch = new CountDownLatch(totalProducedMsgs);
        for (int i = 0; i < consumers.size(); i++) {
            final int consumerCount = i;
            for (int j = 0; j < totalProducedMsgs; j++) {
                consumers.get(i).receiveAsync().thenAccept(m -> {
                    result.add(m.getMessageId());
                    try {
                        consumers.get(consumerCount).acknowledge(m);
                    } catch (PulsarClientException e) {
                        fail("failed to ack msg", e);
                    }
                    latch.countDown();
                });
            }
        }
        latch.await();
        // total received-messages should match to produced messages (it may have duplicate messages)
        assertTrue(result.size() >= totalProducedMsgs);
        producer.close();
        consumers.forEach(c -> {
            try {
                c.close();
            } catch (PulsarClientException e) {
            }
        });
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 14 with ConsumerImpl

use of org.apache.pulsar.client.impl.ConsumerImpl in project incubator-pulsar by apache.

the class NonPersistentTopicTest method testReplicator.

/**
 * verifies that non-persistent topic replicates using replicator
 */
@Test
public void testReplicator() throws Exception {
    ReplicationClusterManager replication = new ReplicationClusterManager();
    replication.setupReplicationCluster();
    try {
        final String globalTopicName = "non-persistent://pulsar/global/ns/nonPersistentTopic";
        final int timeWaitToSync = 100;
        NonPersistentTopicStats stats;
        SubscriptionStats subStats;
        PulsarClient client1 = PulsarClient.builder().serviceUrl(replication.url1.toString()).build();
        PulsarClient client2 = PulsarClient.builder().serviceUrl(replication.url2.toString()).build();
        PulsarClient client3 = PulsarClient.builder().serviceUrl(replication.url3.toString()).build();
        ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) client1.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        ConsumerImpl<byte[]> consumer2 = (ConsumerImpl<byte[]>) client1.newConsumer().topic(globalTopicName).subscriptionName("subscriber-2").subscribe();
        ConsumerImpl<byte[]> repl2Consumer = (ConsumerImpl<byte[]>) client2.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        ConsumerImpl<byte[]> repl3Consumer = (ConsumerImpl<byte[]>) client3.newConsumer().topic(globalTopicName).subscriptionName("subscriber-1").subscribe();
        Producer<byte[]> producer = client1.newProducer().topic(globalTopicName).create();
        Thread.sleep(timeWaitToSync);
        PulsarService replicationPulasr = replication.pulsar1;
        // Replicator for r1 -> r2,r3
        NonPersistentTopic topicRef = (NonPersistentTopic) replication.pulsar1.getBrokerService().getTopicReference(globalTopicName);
        NonPersistentReplicator replicatorR2 = (NonPersistentReplicator) topicRef.getPersistentReplicator("r2");
        NonPersistentReplicator replicatorR3 = (NonPersistentReplicator) topicRef.getPersistentReplicator("r3");
        assertNotNull(topicRef);
        assertNotNull(replicatorR2);
        assertNotNull(replicatorR3);
        rolloverPerIntervalStats(replicationPulasr);
        stats = topicRef.getStats();
        subStats = stats.getSubscriptions().values().iterator().next();
        // subscription stats
        assertEquals(stats.getSubscriptions().keySet().size(), 2);
        assertEquals(subStats.consumers.size(), 1);
        Thread.sleep(timeWaitToSync);
        int totalProducedMessages = 100;
        for (int i = 0; i < totalProducedMessages; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (1) consume by consumer1
        Message<?> msg = null;
        Set<String> messageSet = Sets.newHashSet();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = consumer1.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (2) consume by consumer2
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = consumer2.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (3) consume by repl2consumer
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = repl2Consumer.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        // (4) consume by repl3consumer
        messageSet.clear();
        for (int i = 0; i < totalProducedMessages; i++) {
            msg = repl3Consumer.receive(300, TimeUnit.MILLISECONDS);
            if (msg != null) {
                String receivedMessage = new String(msg.getData());
                testMessageOrderAndDuplicates(messageSet, receivedMessage, "my-message-" + i);
            } else {
                break;
            }
        }
        assertEquals(messageSet.size(), totalProducedMessages);
        Thread.sleep(timeWaitToSync);
        rolloverPerIntervalStats(replicationPulasr);
        stats = topicRef.getStats();
        subStats = stats.getSubscriptions().values().iterator().next();
        assertTrue(subStats.msgRateOut > 0);
        assertEquals(subStats.consumers.size(), 1);
        assertTrue(subStats.msgThroughputOut > 0);
        // consumer stats
        assertTrue(subStats.consumers.get(0).msgRateOut > 0.0);
        assertTrue(subStats.consumers.get(0).msgThroughputOut > 0.0);
        assertEquals(subStats.msgRateRedeliver, 0.0);
        producer.close();
        consumer1.close();
        repl2Consumer.close();
        repl3Consumer.close();
        client1.close();
        client2.close();
        client3.close();
    } finally {
        replication.shutdownReplicationCluster();
    }
}