Examples with Message org.apache.pulsar.client.api.Message used on opensource projects

Example 61 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testShouldNotBlockConsumerIfRedeliverBeforeReceive.

@Test
public void testShouldNotBlockConsumerIfRedeliverBeforeReceive() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    int totalReceiveMsg = 0;
    try {
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 100;
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setAckTimeout(1, TimeUnit.SECONDS);
        conf.setSubscriptionType(SubscriptionType.Shared);
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) wait for consumer to receive messages
        Thread.sleep(1000);
        assertEquals(consumer.numMessagesInQueue(), receiverQueueSize);
        // (3) wait for messages to expire, we should've received more
        Thread.sleep(2000);
        assertEquals(consumer.numMessagesInQueue(), receiverQueueSize);
        for (int i = 0; i < totalProducedMsgs; i++) {
            Message msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                consumer.acknowledge(msg);
                totalReceiveMsg++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // total received-messages should match to produced messages
        assertEquals(totalProducedMsgs, totalReceiveMsg);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 62 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testSendBigMessageSizeButCompressed.

/**
 * Verifies non-batch message size being validated after performing compression while batch-messaging validates
 * before compression of message
 *
 * <pre>
 * send msg with size > MAX_SIZE (5 MB)
 * a. non-batch with compression: pass
 * b. batch-msg with compression: fail
 * c. non-batch w/o  compression: fail
 * d. non-batch with compression, consumer consume: pass
 * </pre>
 *
 * @throws Exception
 */
@Test
public void testSendBigMessageSizeButCompressed() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final String topic = "persistent://my-property/use/my-ns/bigMsg";
    // (a) non-batch msg with compression
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(CompressionType.LZ4);
    Producer producer = pulsarClient.createProducer(topic, producerConf);
    Message message = MessageBuilder.create().setContent(new byte[PulsarDecoder.MaxMessageSize + 1]).build();
    producer.send(message);
    producer.close();
    // (b) batch-msg
    producerConf = new ProducerConfiguration();
    producerConf.setBatchingEnabled(true);
    producerConf.setCompressionType(CompressionType.LZ4);
    producer = pulsarClient.createProducer(topic, producerConf);
    message = MessageBuilder.create().setContent(new byte[PulsarDecoder.MaxMessageSize + 1]).build();
    try {
        producer.send(message);
        fail("Should have thrown exception");
    } catch (PulsarClientException.InvalidMessageException e) {
    // OK
    }
    producer.close();
    // (c) non-batch msg without compression
    producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(CompressionType.NONE);
    producer = pulsarClient.createProducer(topic, producerConf);
    message = MessageBuilder.create().setContent(new byte[PulsarDecoder.MaxMessageSize + 1]).build();
    try {
        producer.send(message);
        fail("Should have thrown exception");
    } catch (PulsarClientException.InvalidMessageException e) {
    // OK
    }
    producer.close();
    // (d) non-batch msg with compression and try to consume message
    producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(CompressionType.LZ4);
    producer = pulsarClient.createProducer(topic, producerConf);
    Consumer consumer = pulsarClient.subscribe(topic, "sub1");
    byte[] content = new byte[PulsarDecoder.MaxMessageSize + 10];
    message = MessageBuilder.create().setContent(content).build();
    producer.send(message);
    assertEquals(consumer.receive().getData(), content);
    producer.close();
    consumer.close();
}

Example 63 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testSharedSamePriorityConsumer.

/**
 * <pre>
 * Verifies Dispatcher dispatches messages properly with shared-subscription consumers with combination of blocked
 * and unblocked consumers.
 *
 * 1. Dispatcher will have 5 consumers : c1, c2, c3, c4, c5.
 *      Out of which : c1,c2,c4,c5 will be blocked due to MaxUnackedMessages limit.
 * 2. So, dispatcher should moves round-robin and make sure it delivers unblocked consumer : c3
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 5000)
public void testSharedSamePriorityConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    ConsumerConfiguration conf1 = new ConsumerConfiguration();
    conf1.setSubscriptionType(SubscriptionType.Shared);
    final int queueSize = 5;
    conf1.setReceiverQueueSize(queueSize);
    int maxUnAckMsgs = pulsar.getConfiguration().getMaxConcurrentLookupRequest();
    pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(queueSize);
    Consumer c1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    Consumer c2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic2", producerConf);
    List<Future<MessageId>> futures = Lists.newArrayList();
    // Asynchronously produce messages
    final int totalPublishMessages = 500;
    for (int i = 0; i < totalPublishMessages; i++) {
        final String message = "my-message-" + i;
        Future<MessageId> future = producer.sendAsync(message.getBytes());
        futures.add(future);
    }
    log.info("Waiting for async publish to complete");
    for (Future<MessageId> future : futures) {
        future.get();
    }
    List<Message> messages = Lists.newArrayList();
    // let consumer1 and consumer2 cosume messages up to the queue will be full
    for (int i = 0; i < totalPublishMessages; i++) {
        Message msg = c1.receive(500, TimeUnit.MILLISECONDS);
        if (msg != null) {
            messages.add(msg);
        } else {
            break;
        }
    }
    for (int i = 0; i < totalPublishMessages; i++) {
        Message msg = c2.receive(500, TimeUnit.MILLISECONDS);
        if (msg != null) {
            messages.add(msg);
        } else {
            break;
        }
    }
    Assert.assertEquals(queueSize * 2, messages.size());
    // create new consumers with the same priority
    Consumer c3 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    Consumer c4 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    Consumer c5 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    for (int i = 0; i < totalPublishMessages; i++) {
        Message msg = c4.receive(500, TimeUnit.MILLISECONDS);
        if (msg != null) {
            messages.add(msg);
        } else {
            break;
        }
    }
    for (int i = 0; i < totalPublishMessages; i++) {
        Message msg = c5.receive(500, TimeUnit.MILLISECONDS);
        if (msg != null) {
            messages.add(msg);
        } else {
            break;
        }
    }
    for (int i = 0; i < totalPublishMessages; i++) {
        Message msg = c3.receive(500, TimeUnit.MILLISECONDS);
        if (msg != null) {
            messages.add(msg);
            c3.acknowledge(msg);
        } else {
            break;
        }
    }
    // total messages must be consumed by all consumers
    Assert.assertEquals(messages.size(), totalPublishMessages);
    // Asynchronously acknowledge upto and including the last message
    producer.close();
    c1.close();
    c2.close();
    c3.close();
    c4.close();
    c5.close();
    pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnAckMsgs);
    log.info("-- Exiting {} test --", methodName);
}

Example 64 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testMutlipleSharedConsumerBlockingWithUnAckedMessages.

/**
 * Verify: Consumer1 which doesn't send ack will not impact Consumer2 which sends ack for consumed message.
 *
 * @param batchMessageDelayMs
 * @throws Exception
 */
@Test
public void testMutlipleSharedConsumerBlockingWithUnAckedMessages() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int maxUnackedMessages = 20;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 100;
        int totalReceiveMessages = 0;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnackedMessages);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) Consumer1: consume without ack:
        // try to consume messages: but will be able to consume number of messages = maxUnackedMessages
        Message msg = null;
        List<Message> messages = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer1.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                totalReceiveMessages++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
        assertEquals(messages.size(), maxUnackedMessages);
        // (3.1) Consumer2 will start consuming messages without ack: it should stop after maxUnackedMessages
        messages.clear();
        for (int i = 0; i < totalProducedMsgs - maxUnackedMessages; i++) {
            msg = consumer2.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                totalReceiveMessages++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        assertEquals(messages.size(), maxUnackedMessages);
        // (3.2) ack for all maxUnackedMessages
        messages.forEach(m -> {
            try {
                consumer2.acknowledge(m);
            } catch (PulsarClientException e) {
                fail("shouldn't have failed ", e);
            }
        });
        // (4) Consumer2 consumer and ack: so it should consume all remaining messages
        messages.clear();
        for (int i = 0; i < totalProducedMsgs - (2 * maxUnackedMessages); i++) {
            msg = consumer2.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                totalReceiveMessages++;
                consumer2.acknowledge(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // verify total-consumer messages = total-produce messages
        assertEquals(totalProducedMsgs, totalReceiveMessages);
        producer.close();
        consumer1.close();
        consumer2.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 65 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testConcurrentConsumerReceiveWhileReconnect.

// This is to test that the flow control counter doesn't get corrupted while concurrent receives during
// reconnections
@Test(dataProvider = "batch")
public void testConcurrentConsumerReceiveWhileReconnect(int batchMessageDelayMs) throws Exception {
    final int recvQueueSize = 100;
    final int numConsumersThreads = 10;
    final ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(recvQueueSize);
    String subName = UUID.randomUUID().toString();
    final Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic7", subName, conf);
    ExecutorService executor = Executors.newCachedThreadPool();
    final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    // there will be 10 threads calling receive() from the same consumer and will block
    Thread.sleep(100);
    // we restart the broker to reconnect
    restartBroker();
    Thread.sleep(2000);
    // publish 100 messages so that the consumers blocked on receive() will now get the messages
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic7", producerConf);
    for (int i = 0; i < recvQueueSize; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Thread.sleep(500);
    ConsumerImpl consumerImpl = (ConsumerImpl) consumer;
    // The available permits should be 10 and num messages in the queue should be 90
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
    barrier.reset();
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    Thread.sleep(100);
    // The available permits should be 20 and num messages in the queue should be 80
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads * 2);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - (numConsumersThreads * 2));
    // clear the queue
    while (true) {
        Message msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg == null) {
            break;
        }
    }
    // The available permits should be 0 and num messages in the queue should be 0
    Assert.assertEquals(consumerImpl.getAvailablePermits(), 0);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), 0);
    barrier.reset();
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                consumer.receive();
                return null;
            }
        });
    }
    barrier.await();
    // we again make 10 threads call receive() and get blocked
    Thread.sleep(100);
    restartBroker();
    Thread.sleep(2000);
    // The available permits should be 10 and num messages in the queue should be 90
    Assert.assertEquals(consumerImpl.getAvailablePermits(), numConsumersThreads);
    Assert.assertEquals(consumerImpl.numMessagesInQueue(), recvQueueSize - numConsumersThreads);
    consumer.close();
}