Examples with MessageId org.apache.pulsar.client.api.MessageId used on opensource projects

Example 46 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class PersistentQueueE2ETest method testRoundRobinBatchDistribution.

// this test is good to have to see the distribution, but every now and then it gets slightly different than the
// expected numbers. keeping this disabled to not break the build, but nevertheless this gives good insight into
// how the round robin distribution algorithm is behaving
@Test(enabled = false)
public void testRoundRobinBatchDistribution() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic5";
    final String subName = "sub5";
    final int numMsgs = 137;
    /* some random number different than default batch size of 100 */
    final AtomicInteger counter1 = new AtomicInteger(0);
    final AtomicInteger counter2 = new AtomicInteger(0);
    final AtomicInteger counter3 = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(numMsgs * 3);
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).receiverQueueSize(10).subscriptionType(SubscriptionType.Shared);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().messageListener((consumer, msg) -> {
        try {
            counter1.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    }).subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().messageListener((consumer, msg) -> {
        try {
            counter2.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    }).subscribe();
    Consumer<byte[]> consumer3 = consumerBuilder.clone().messageListener((consumer, msg) -> {
        try {
            counter1.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    }).subscribe();
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < numMsgs * 3; i++) {
        String message = "msg-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    producer.close();
    latch.await(1, TimeUnit.SECONDS);
    /*
         * total messages = 137 * 3 = 411 Each consumer has 10 permits. There will be 411 / 3*10 = 13 full distributions
         * i.e. each consumer will get 130 messages. In the 14th round, the balance is 411 - 130*3 = 21. Two consumers
         * will get another batch of 10 messages (Total: 140) and the 3rd one will get the last one (Total: 131)
         */
    assertTrue(CollectionUtils.subtract(Lists.newArrayList(140, 140, 131), Lists.newArrayList(counter1.get(), counter2.get(), counter3.get())).isEmpty());
    consumer1.close();
    consumer2.close();
    consumer3.close();
    admin.persistentTopics().delete(topicName);
}

Example 47 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class PersistentQueueE2ETest method testConsumersWithDifferentPermits.

@Test
public void testConsumersWithDifferentPermits() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic4";
    final String subName = "sub4";
    final int numMsgs = 10000;
    final AtomicInteger msgCountConsumer1 = new AtomicInteger(0);
    final AtomicInteger msgCountConsumer2 = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(numMsgs);
    int recvQ1 = 10;
    Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ1).messageListener((consumer, msg) -> {
        msgCountConsumer1.incrementAndGet();
        try {
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (PulsarClientException e) {
            fail("Should not fail");
        }
    }).subscribe();
    int recvQ2 = 1;
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ2).messageListener((consumer, msg) -> {
        msgCountConsumer2.incrementAndGet();
        try {
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (PulsarClientException e) {
            fail("Should not fail");
        }
    }).subscribe();
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).maxPendingMessages(numMsgs + 1).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "msg-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    producer.close();
    latch.await(5, TimeUnit.SECONDS);
    assertEquals(msgCountConsumer1.get(), numMsgs - numMsgs / (recvQ1 + recvQ2), numMsgs * 0.1);
    assertEquals(msgCountConsumer2.get(), numMsgs / (recvQ1 + recvQ2), numMsgs * 0.1);
    consumer1.close();
    consumer2.close();
    admin.persistentTopics().delete(topicName);
}

Example 48 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testSendTimeout.

@Test(dataProvider = "batch")
public void testSendTimeout(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    ConsumerConfiguration consumerConf = new ConsumerConfiguration();
    consumerConf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic5", "my-subscriber-name", consumerConf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(2 * batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    producerConf.setSendTimeout(1, TimeUnit.SECONDS);
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic5", producerConf);
    final String message = "my-message";
    // Trigger the send timeout
    stopBroker();
    Future<MessageId> future = producer.sendAsync(message.getBytes());
    try {
        future.get();
        Assert.fail("Send operation should have failed");
    } catch (ExecutionException e) {
    // Expected
    }
    startBroker();
    // We should not have received any message
    Message msg = consumer.receive(3, TimeUnit.SECONDS);
    Assert.assertNull(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 49 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testSharedSamePriorityConsumer.

/**
 * <pre>
 * Verifies Dispatcher dispatches messages properly with shared-subscription consumers with combination of blocked
 * and unblocked consumers.
 *
 * 1. Dispatcher will have 5 consumers : c1, c2, c3, c4, c5.
 *      Out of which : c1,c2,c4,c5 will be blocked due to MaxUnackedMessages limit.
 * 2. So, dispatcher should moves round-robin and make sure it delivers unblocked consumer : c3
 * </pre>
 *
 * @throws Exception
 */
@Test(timeOut = 5000)
public void testSharedSamePriorityConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    ConsumerConfiguration conf1 = new ConsumerConfiguration();
    conf1.setSubscriptionType(SubscriptionType.Shared);
    final int queueSize = 5;
    conf1.setReceiverQueueSize(queueSize);
    int maxUnAckMsgs = pulsar.getConfiguration().getMaxConcurrentLookupRequest();
    pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(queueSize);
    Consumer c1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    Consumer c2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic2", producerConf);
    List<Future<MessageId>> futures = Lists.newArrayList();
    // Asynchronously produce messages
    final int totalPublishMessages = 500;
    for (int i = 0; i < totalPublishMessages; i++) {
        final String message = "my-message-" + i;
        Future<MessageId> future = producer.sendAsync(message.getBytes());
        futures.add(future);
    }
    log.info("Waiting for async publish to complete");
    for (Future<MessageId> future : futures) {
        future.get();
    }
    List<Message> messages = Lists.newArrayList();
    // let consumer1 and consumer2 cosume messages up to the queue will be full
    for (int i = 0; i < totalPublishMessages; i++) {
        Message msg = c1.receive(500, TimeUnit.MILLISECONDS);
        if (msg != null) {
            messages.add(msg);
        } else {
            break;
        }
    }
    for (int i = 0; i < totalPublishMessages; i++) {
        Message msg = c2.receive(500, TimeUnit.MILLISECONDS);
        if (msg != null) {
            messages.add(msg);
        } else {
            break;
        }
    }
    Assert.assertEquals(queueSize * 2, messages.size());
    // create new consumers with the same priority
    Consumer c3 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    Consumer c4 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    Consumer c5 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    for (int i = 0; i < totalPublishMessages; i++) {
        Message msg = c4.receive(500, TimeUnit.MILLISECONDS);
        if (msg != null) {
            messages.add(msg);
        } else {
            break;
        }
    }
    for (int i = 0; i < totalPublishMessages; i++) {
        Message msg = c5.receive(500, TimeUnit.MILLISECONDS);
        if (msg != null) {
            messages.add(msg);
        } else {
            break;
        }
    }
    for (int i = 0; i < totalPublishMessages; i++) {
        Message msg = c3.receive(500, TimeUnit.MILLISECONDS);
        if (msg != null) {
            messages.add(msg);
            c3.acknowledge(msg);
        } else {
            break;
        }
    }
    // total messages must be consumed by all consumers
    Assert.assertEquals(messages.size(), totalPublishMessages);
    // Asynchronously acknowledge upto and including the last message
    producer.close();
    c1.close();
    c2.close();
    c3.close();
    c4.close();
    c5.close();
    pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnAckMsgs);
    log.info("-- Exiting {} test --", methodName);
}

Example 50 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testAsyncProducerAndAsyncAck.

@Test(dataProvider = "batch")
public void testAsyncProducerAndAsyncAck(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic2", producerConf);
    List<Future<MessageId>> futures = Lists.newArrayList();
    // Asynchronously produce messages
    for (int i = 0; i < 10; i++) {
        final String message = "my-message-" + i;
        Future<MessageId> future = producer.sendAsync(message.getBytes());
        futures.add(future);
    }
    log.info("Waiting for async publish to complete");
    for (Future<MessageId> future : futures) {
        future.get();
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.info("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Asynchronously acknowledge upto and including the last message
    Future<Void> ackFuture = consumer.acknowledgeCumulativeAsync(msg);
    log.info("Waiting for async ack to complete");
    ackFuture.get();
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}