Search in sources :

Example 26 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testBackoffAndReconnect.

@Test(dataProvider = "batch")
public void testBackoffAndReconnect(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    // Create consumer and producer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic4", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic4", producerConf);
    // Produce messages
    CompletableFuture<MessageId> lastFuture = null;
    for (int i = 0; i < 10; i++) {
        lastFuture = producer.sendAsync(("my-message-" + i).getBytes()).thenApply(msgId -> {
            log.info("Published message id: {}", msgId);
            return msgId;
        });
    }
    lastFuture.get();
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
    }
    // Restart the broker and wait for the backoff to kick in. The client library will try to reconnect, and once
    // the broker is up, the consumer should receive the duplicate messages.
    log.info("-- Restarting broker --");
    restartBroker();
    msg = null;
    log.info("Receiving duplicate messages..");
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
        Assert.assertNotNull(msg, "Message cannot be null");
    }
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
Also used : Producer(org.apache.pulsar.client.api.Producer) LoggerFactory(org.slf4j.LoggerFactory) Test(org.testng.annotations.Test) AfterMethod(org.testng.annotations.AfterMethod) Future(java.util.concurrent.Future) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) EncryptionKeyInfo(org.apache.pulsar.client.api.EncryptionKeyInfo) Assert.assertFalse(org.testng.Assert.assertFalse) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) PulsarDecoder(org.apache.pulsar.common.api.PulsarDecoder) Assert.assertNotEquals(org.testng.Assert.assertNotEquals) CyclicBarrier(java.util.concurrent.CyclicBarrier) ConsumerImpl(org.apache.pulsar.client.impl.ConsumerImpl) MessageBuilder(org.apache.pulsar.client.api.MessageBuilder) CompressionType(org.apache.pulsar.client.api.CompressionType) Mockito.atLeastOnce(org.mockito.Mockito.atLeastOnce) BeforeMethod(org.testng.annotations.BeforeMethod) Set(java.util.Set) UUID(java.util.UUID) ConsumerConfiguration(org.apache.pulsar.client.api.ConsumerConfiguration) Collectors(java.util.stream.Collectors) ProducerConfiguration(org.apache.pulsar.client.api.ProducerConfiguration) Executors(java.util.concurrent.Executors) Sets(com.google.common.collect.Sets) Matchers.any(org.mockito.Matchers.any) MessageIdImpl(org.apache.pulsar.client.impl.MessageIdImpl) CountDownLatch(java.util.concurrent.CountDownLatch) Consumer(org.apache.pulsar.client.api.Consumer) List(java.util.List) PersistentTopic(org.apache.pulsar.broker.service.persistent.PersistentTopic) FutureUtil(org.apache.pulsar.common.util.FutureUtil) Modifier(java.lang.reflect.Modifier) ProducerConsumerBase(org.apache.pulsar.client.api.ProducerConsumerBase) ConsumerCryptoFailureAction(org.apache.pulsar.client.api.ConsumerCryptoFailureAction) TopicName(org.apache.pulsar.common.naming.TopicName) DataProvider(org.testng.annotations.DataProvider) Assert.assertEquals(org.testng.Assert.assertEquals) Callable(java.util.concurrent.Callable) CompletableFuture(java.util.concurrent.CompletableFuture) Message(org.apache.pulsar.client.api.Message) Mockito.spy(org.mockito.Mockito.spy) Lists(com.google.common.collect.Lists) Assert(org.testng.Assert) PulsarClient(org.apache.pulsar.client.api.PulsarClient) ExecutorService(java.util.concurrent.ExecutorService) ManagedLedgerImpl(org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl) Logger(org.slf4j.Logger) Files(java.nio.file.Files) Assert.fail(org.testng.Assert.fail) IOException(java.io.IOException) Field(java.lang.reflect.Field) EntryCacheImpl(org.apache.bookkeeper.mledger.impl.EntryCacheImpl) SubscriptionType(org.apache.pulsar.client.api.SubscriptionType) Mockito.verify(org.mockito.Mockito.verify) ExecutionException(java.util.concurrent.ExecutionException) TimeUnit(java.util.concurrent.TimeUnit) CryptoKeyReader(org.apache.pulsar.client.api.CryptoKeyReader) MessageId(org.apache.pulsar.client.api.MessageId) Paths(java.nio.file.Paths) Assert.assertTrue(org.testng.Assert.assertTrue) Consumer(org.apache.pulsar.client.api.Consumer) Producer(org.apache.pulsar.client.api.Producer) Message(org.apache.pulsar.client.api.Message) ConsumerConfiguration(org.apache.pulsar.client.api.ConsumerConfiguration) ProducerConfiguration(org.apache.pulsar.client.api.ProducerConfiguration) MessageId(org.apache.pulsar.client.api.MessageId) Test(org.testng.annotations.Test)

Example 27 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testEnabledChecksumClient.

@Test
public void testEnabledChecksumClient() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int totalMsg = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    final int batchMessageDelayMs = 300;
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < totalMsg; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}
Also used : Consumer(org.apache.pulsar.client.api.Consumer) Producer(org.apache.pulsar.client.api.Producer) Message(org.apache.pulsar.client.api.Message) ConsumerConfiguration(org.apache.pulsar.client.api.ConsumerConfiguration) ProducerConfiguration(org.apache.pulsar.client.api.ProducerConfiguration) Test(org.testng.annotations.Test)

Example 28 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testConsumerBlockingWithUnAckedMessagesMultipleIteration.

/**
 * Verify: iteration of a. message receive w/o acking b. stop receiving msg c. ack msgs d. started receiving msgs
 *
 * 1. Produce total X (1500) messages 2. Consumer consumes messages without acking until stop receiving from broker
 * due to reaching ack-threshold (500) 3. Consumer acks messages after stop getting messages 4. Consumer again tries
 * to consume messages 5. Consumer should be able to complete consuming all 1500 messages in 3 iteration (1500/500)
 *
 * @throws Exception
 */
@Test
public void testConsumerBlockingWithUnAckedMessagesMultipleIteration() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 500;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 1500;
        // receiver consumes messages in iteration after acknowledging broker
        final int totalReceiveIteration = totalProducedMsgs / unAckedMessagesBufferSize;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        int totalReceivedMessages = 0;
        // (2) Receive Messages
        for (int j = 0; j < totalReceiveIteration; j++) {
            Message msg = null;
            List<Message> messages = Lists.newArrayList();
            for (int i = 0; i < totalProducedMsgs; i++) {
                msg = consumer.receive(1, TimeUnit.SECONDS);
                if (msg != null) {
                    messages.add(msg);
                    log.info("Received message: " + new String(msg.getData()));
                } else {
                    break;
                }
            }
            // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
            assertEquals(messages.size(), unAckedMessagesBufferSize);
            // start acknowledging messages
            messages.forEach(m -> {
                try {
                    consumer.acknowledge(m);
                } catch (PulsarClientException e) {
                    fail("ack failed", e);
                }
            });
            totalReceivedMessages += messages.size();
        }
        // total received-messages should match to produced messages
        assertEquals(totalReceivedMessages, totalProducedMsgs);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}
Also used : Consumer(org.apache.pulsar.client.api.Consumer) Producer(org.apache.pulsar.client.api.Producer) Message(org.apache.pulsar.client.api.Message) ConsumerConfiguration(org.apache.pulsar.client.api.ConsumerConfiguration) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) ProducerConfiguration(org.apache.pulsar.client.api.ProducerConfiguration) PulsarClientException(org.apache.pulsar.client.api.PulsarClientException) IOException(java.io.IOException) ExecutionException(java.util.concurrent.ExecutionException) Test(org.testng.annotations.Test)

Example 29 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class TopicTerminationTest method testSimpleTerminationMessageListener.

@Test(timeOut = 20000)
public void testSimpleTerminationMessageListener() throws Exception {
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    CountDownLatch latch = new CountDownLatch(1);
    org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").messageListener(new MessageListener<byte[]>() {

        @Override
        public void received(Consumer<byte[]> consumer, Message<byte[]> msg) {
        // do nothing
        }

        @Override
        public void reachedEndOfTopic(Consumer<byte[]> consumer) {
            latch.countDown();
            assertTrue(consumer.hasReachedEndOfTopic());
        }
    }).subscribe();
    /* MessageId msgId1 = */
    producer.send("test-msg-1".getBytes());
    /* MessageId msgId2 = */
    producer.send("test-msg-2".getBytes());
    MessageId msgId3 = producer.send("test-msg-3".getBytes());
    consumer.acknowledgeCumulative(msgId3);
    Thread.sleep(100);
    assertFalse(consumer.hasReachedEndOfTopic());
    MessageId lastMessageId = admin.persistentTopics().terminateTopicAsync(topicName).get();
    assertEquals(lastMessageId, msgId3);
    assertTrue(latch.await(3, TimeUnit.SECONDS));
    assertTrue(consumer.hasReachedEndOfTopic());
}
Also used : Consumer(org.apache.pulsar.client.api.Consumer) Message(org.apache.pulsar.client.api.Message) MessageListener(org.apache.pulsar.client.api.MessageListener) CountDownLatch(java.util.concurrent.CountDownLatch) MessageId(org.apache.pulsar.client.api.MessageId) Test(org.testng.annotations.Test)

Example 30 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class AdminApiTest method persistentTopics.

@Test(dataProvider = "topicName")
public void persistentTopics(String topicName) throws Exception {
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    final String persistentTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
    // Force to create a topic
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList("persistent://prop-xyz/use/ns1/" + topicName));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS).build();
    Consumer<byte[]> consumer = client.newConsumer().topic(persistentTopicName).subscriptionName("my-sub").subscriptionType(SubscriptionType.Exclusive).subscribe();
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 10);
    PersistentTopicStats topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
    assertEquals(topicStats.publishers.size(), 0);
    PersistentTopicInternalStats internalStats = admin.persistentTopics().getInternalStats(persistentTopicName);
    assertEquals(internalStats.cursors.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    List<Message<byte[]>> messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 3);
    assertEquals(messages.size(), 3);
    for (int i = 0; i < 3; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 15);
    assertEquals(messages.size(), 10);
    for (int i = 0; i < 10; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    admin.persistentTopics().skipMessages(persistentTopicName, "my-sub", 5);
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 5);
    admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
    consumer.close();
    client.close();
    admin.persistentTopics().deleteSubscription(persistentTopicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList());
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet());
    assertEquals(topicStats.publishers.size(), 0);
    try {
        admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    } catch (NotFoundException e) {
    }
    admin.persistentTopics().delete(persistentTopicName);
    try {
        admin.persistentTopics().delete(persistentTopicName);
        fail("Should have received 404");
    } catch (NotFoundException e) {
    }
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
}
Also used : Message(org.apache.pulsar.client.api.Message) PersistentTopicInternalStats(org.apache.pulsar.common.policies.data.PersistentTopicInternalStats) NotFoundException(org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException) PulsarClient(org.apache.pulsar.client.api.PulsarClient) PersistentTopicStats(org.apache.pulsar.common.policies.data.PersistentTopicStats) URL(java.net.URL) Test(org.testng.annotations.Test) MockedPulsarServiceBaseTest(org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest)

Aggregations

Message (org.apache.pulsar.client.api.Message)72 Test (org.testng.annotations.Test)59 Producer (org.apache.pulsar.client.api.Producer)38 Consumer (org.apache.pulsar.client.api.Consumer)35 PulsarClientException (org.apache.pulsar.client.api.PulsarClientException)28 MessageId (org.apache.pulsar.client.api.MessageId)27 ProducerConfiguration (org.apache.pulsar.client.api.ProducerConfiguration)27 ConsumerConfiguration (org.apache.pulsar.client.api.ConsumerConfiguration)25 ExecutionException (java.util.concurrent.ExecutionException)20 IOException (java.io.IOException)19 CompletableFuture (java.util.concurrent.CompletableFuture)16 ExecutorService (java.util.concurrent.ExecutorService)15 MessageIdImpl (org.apache.pulsar.client.impl.MessageIdImpl)13 Map (java.util.Map)12 TimeUnit (java.util.concurrent.TimeUnit)12 List (java.util.List)11 PrepareForTest (org.powermock.core.classloader.annotations.PrepareForTest)11 PulsarClient (org.apache.pulsar.client.api.PulsarClient)10 Logger (org.slf4j.Logger)10 LoggerFactory (org.slf4j.LoggerFactory)10