Search in sources :

Example 1 with ConsumerConfiguration

use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause.

/**
 * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
 * blocked due to unacked messsages
 *
 * Usecase: produce message with 10ms interval: so, consumer can consume only 10 messages without acking
 *
 * @throws Exception
 */
@Test
public void testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 10;
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 20;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
            Thread.sleep(10);
        }
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages1 = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages1.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client should not receive all produced messages and should be blocked due to unack-messages
        assertEquals(messages1.size(), unAckedMessagesBufferSize);
        Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
            return (MessageIdImpl) m.getMessageId();
        }).collect(Collectors.toSet());
        // (3) redeliver all consumed messages
        consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
        Thread.sleep(1000);
        Set<MessageIdImpl> messages2 = Sets.newHashSet();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages2.add((MessageIdImpl) msg.getMessageId());
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        assertEquals(messages1.size(), messages2.size());
        // (4) Verify: redelivered all previous unacked-consumed messages
        messages2.removeAll(redeliveryMessages);
        assertEquals(messages2.size(), 0);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}
Also used : Producer(org.apache.pulsar.client.api.Producer) LoggerFactory(org.slf4j.LoggerFactory) Test(org.testng.annotations.Test) AfterMethod(org.testng.annotations.AfterMethod) Future(java.util.concurrent.Future) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) EncryptionKeyInfo(org.apache.pulsar.client.api.EncryptionKeyInfo) Assert.assertFalse(org.testng.Assert.assertFalse) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) PulsarDecoder(org.apache.pulsar.common.api.PulsarDecoder) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) CyclicBarrier(java.util.concurrent.CyclicBarrier) ConsumerImpl(org.apache.pulsar.client.impl.ConsumerImpl) MessageBuilder(org.apache.pulsar.client.api.MessageBuilder) CompressionType(org.apache.pulsar.client.api.CompressionType) Mockito.atLeastOnce(org.mockito.Mockito.atLeastOnce) BeforeMethod(org.testng.annotations.BeforeMethod) Set(java.util.Set) UUID(java.util.UUID) ConsumerConfiguration(org.apache.pulsar.client.api.ConsumerConfiguration) Collectors(java.util.stream.Collectors) ProducerConfiguration(org.apache.pulsar.client.api.ProducerConfiguration) Executors(java.util.concurrent.Executors) Sets(com.google.common.collect.Sets) Matchers.any(org.mockito.Matchers.any) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) CountDownLatch(java.util.concurrent.CountDownLatch) Consumer(org.apache.pulsar.client.api.Consumer) List(java.util.List) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) FutureUtil(org.apache.pulsar.common.util.FutureUtil) Modifier(java.lang.reflect.Modifier) ProducerConsumerBase(org.apache.pulsar.client.api.ProducerConsumerBase) ConsumerCryptoFailureAction(org.apache.pulsar.client.api.ConsumerCryptoFailureAction) TopicName(org.apache.pulsar.common.naming.TopicName) DataProvider(org.testng.annotations.DataProvider) Assert.assertEquals(org.testng.Assert.assertEquals) Callable(java.util.concurrent.Callable) CompletableFuture(java.util.concurrent.CompletableFuture) Message(org.apache.pulsar.client.api.Message) Mockito.spy(org.mockito.Mockito.spy) Lists(com.google.common.collect.Lists) Assert(org.testng.Assert) PulsarClient(org.apache.pulsar.client.api.PulsarClient) ExecutorService(java.util.concurrent.ExecutorService) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) Logger(org.slf4j.Logger) Files(java.nio.file.Files) Assert.fail(org.testng.Assert.fail) IOException(java.io.IOException) Field(java.lang.reflect.Field) EntryCacheImpl(org.apache.bookkeeper.mledger.impl.EntryCacheImpl) SubscriptionType(org.apache.pulsar.client.api.SubscriptionType) Mockito.verify(org.mockito.Mockito.verify) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) CryptoKeyReader(org.apache.pulsar.client.api.CryptoKeyReader) MessageId(org.apache.pulsar.client.api.MessageId) Paths(java.nio.file.Paths) Assert.assertTrue(org.testng.Assert.assertTrue) ConsumerImpl(org.apache.pulsar.client.impl.ConsumerImpl) Producer(org.apache.pulsar.client.api.Producer) Message(org.apache.pulsar.client.api.Message) ConsumerConfiguration(org.apache.pulsar.client.api.ConsumerConfiguration) ProducerConfiguration(org.apache.pulsar.client.api.ProducerConfiguration) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) Test(org.testng.annotations.Test)

Example 2 with ConsumerConfiguration

use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testAsyncProducerAndConsumer.

@Test(timeOut = 2000)
public void testAsyncProducerAndConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int totalMsg = 100;
    final Set<String> produceMsgs = Sets.newHashSet();
    final Set<String> consumeMsgs = Sets.newHashSet();
    final ProducerConfiguration producerConf = new ProducerConfiguration();
    final ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
    // produce message
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        produceMsgs.add(message);
    }
    log.info(" start receiving messages :");
    CountDownLatch latch = new CountDownLatch(totalMsg);
    // receive messages
    ExecutorService executor = Executors.newFixedThreadPool(1);
    receiveAsync(consumer, totalMsg, 0, latch, consumeMsgs, executor);
    latch.await();
    // verify message produced correctly
    assertEquals(produceMsgs.size(), totalMsg);
    // verify produced and consumed messages must be exactly same
    produceMsgs.removeAll(consumeMsgs);
    assertTrue(produceMsgs.isEmpty());
    producer.close();
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
Also used : Consumer(org.apache.pulsar.client.api.Consumer) Producer(org.apache.pulsar.client.api.Producer) ConsumerConfiguration(org.apache.pulsar.client.api.ConsumerConfiguration) ExecutorService(java.util.concurrent.ExecutorService) ProducerConfiguration(org.apache.pulsar.client.api.ProducerConfiguration) CountDownLatch(java.util.concurrent.CountDownLatch) Test(org.testng.annotations.Test)

Example 3 with ConsumerConfiguration

use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testBlockUnackConsumerAckByDifferentConsumer.

/**
 * Verify: Consumer2 sends ack of Consumer1 and consumer1 should be unblock if it is blocked due to unack-messages
 *
 * @param batchMessageDelayMs
 * @throws Exception
 */
@Test
public void testBlockUnackConsumerAckByDifferentConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int maxUnackedMessages = 20;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 100;
        int totalReceiveMessages = 0;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnackedMessages);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) Consumer1: consume without ack:
        // try to consume messages: but will be able to consume number of messages = maxUnackedMessages
        Message msg = null;
        List<Message> messages = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer1.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                totalReceiveMessages++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // consumer1
        assertEquals(messages.size(), maxUnackedMessages);
        // (3) ack for all UnackedMessages from consumer2
        messages.forEach(m -> {
            try {
                consumer2.acknowledge(m);
            } catch (PulsarClientException e) {
                fail("shouldn't have failed ", e);
            }
        });
        // (4) consumer1 will consumer remaining msgs and consumer2 will ack those messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer1.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                totalReceiveMessages++;
                consumer2.acknowledge(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer2.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                totalReceiveMessages++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // verify total-consumer messages = total-produce messages
        assertEquals(totalProducedMsgs, totalReceiveMessages);
        producer.close();
        consumer1.close();
        consumer2.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}
Also used : Consumer(org.apache.pulsar.client.api.Consumer) Producer(org.apache.pulsar.client.api.Producer) Message(org.apache.pulsar.client.api.Message) ConsumerConfiguration(org.apache.pulsar.client.api.ConsumerConfiguration) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) ProducerConfiguration(org.apache.pulsar.client.api.ProducerConfiguration) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) Test(org.testng.annotations.Test)

Example 4 with ConsumerConfiguration

use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testUnackBlockRedeliverMessages.

@Test
public void testUnackBlockRedeliverMessages() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    int totalReceiveMsg = 0;
    try {
        final int unAckedMessagesBufferSize = 20;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 100;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                totalReceiveMsg++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        consumer.redeliverUnacknowledgedMessages();
        Thread.sleep(1000);
        int alreadyConsumedMessages = messages.size();
        messages.clear();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                consumer.acknowledge(msg);
                totalReceiveMsg++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // total received-messages should match to produced messages
        assertEquals(totalProducedMsgs + alreadyConsumedMessages, totalReceiveMsg);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}
Also used : ConsumerImpl(org.apache.pulsar.client.impl.ConsumerImpl) Producer(org.apache.pulsar.client.api.Producer) Message(org.apache.pulsar.client.api.Message) ConsumerConfiguration(org.apache.pulsar.client.api.ConsumerConfiguration) ProducerConfiguration(org.apache.pulsar.client.api.ProducerConfiguration) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) Test(org.testng.annotations.Test)

Example 5 with ConsumerConfiguration

use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testSharedConsumerAckDifferentConsumer.

/**
 * consume message from consumer1 and send acknowledgement from different consumer subscribed under same
 * subscription-name
 *
 * @throws Exception
 */
@Test(timeOut = 30000)
public void testSharedConsumerAckDifferentConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(1);
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
    Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<Message> consumerMsgSet1 = Sets.newHashSet();
    Set<Message> consumerMsgSet2 = Sets.newHashSet();
    for (int i = 0; i < 5; i++) {
        msg = consumer1.receive();
        consumerMsgSet1.add(msg);
        msg = consumer2.receive();
        consumerMsgSet2.add(msg);
    }
    consumerMsgSet1.stream().forEach(m -> {
        try {
            consumer2.acknowledge(m);
        } catch (PulsarClientException e) {
            fail();
        }
    });
    consumerMsgSet2.stream().forEach(m -> {
        try {
            consumer1.acknowledge(m);
        } catch (PulsarClientException e) {
            fail();
        }
    });
    consumer1.redeliverUnacknowledgedMessages();
    consumer2.redeliverUnacknowledgedMessages();
    try {
        if (consumer1.receive(100, TimeUnit.MILLISECONDS) != null || consumer2.receive(100, TimeUnit.MILLISECONDS) != null) {
            fail();
        }
    } finally {
        consumer1.close();
        consumer2.close();
    }
    log.info("-- Exiting {} test --", methodName);
}
Also used : Consumer(org.apache.pulsar.client.api.Consumer) Producer(org.apache.pulsar.client.api.Producer) Message(org.apache.pulsar.client.api.Message) ConsumerConfiguration(org.apache.pulsar.client.api.ConsumerConfiguration) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) ProducerConfiguration(org.apache.pulsar.client.api.ProducerConfiguration) Test(org.testng.annotations.Test)

Aggregations

ConsumerConfiguration (org.apache.pulsar.client.api.ConsumerConfiguration)36 Test (org.testng.annotations.Test)32 Producer (org.apache.pulsar.client.api.Producer)29 ProducerConfiguration (org.apache.pulsar.client.api.ProducerConfiguration)29 Consumer (org.apache.pulsar.client.api.Consumer)28 Message (org.apache.pulsar.client.api.Message)25 PulsarClientException (org.apache.pulsar.client.api.PulsarClientException)15 IOException (java.io.IOException)14 ExecutionException (java.util.concurrent.ExecutionException)14 CompletableFuture (java.util.concurrent.CompletableFuture)9 MessageId (org.apache.pulsar.client.api.MessageId)9 Future (java.util.concurrent.Future)7 ConsumerImpl (org.apache.pulsar.client.impl.ConsumerImpl)7 Map (java.util.Map)6 CountDownLatch (java.util.concurrent.CountDownLatch)6 ExecutorService (java.util.concurrent.ExecutorService)6 CryptoKeyReader (org.apache.pulsar.client.api.CryptoKeyReader)6 EncryptionKeyInfo (org.apache.pulsar.client.api.EncryptionKeyInfo)6 Field (java.lang.reflect.Field)5 ManagedLedgerImpl (org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl)5