Examples with Message org.apache.pulsar.client.api.Message used on opensource projects

Example 51 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class ConsumerImpl method redeliverUnacknowledgedMessages.

@Override
public void redeliverUnacknowledgedMessages(Set<MessageId> messageIds) {
    checkArgument(messageIds.stream().findFirst().get() instanceof MessageIdImpl);
    if (conf.getSubscriptionType() != SubscriptionType.Shared) {
        // We cannot redeliver single messages if subscription type is not Shared
        redeliverUnacknowledgedMessages();
        return;
    }
    ClientCnx cnx = cnx();
    if (isConnected() && cnx.getRemoteEndpointProtocolVersion() >= ProtocolVersion.v2.getNumber()) {
        int messagesFromQueue = removeExpiredMessagesFromQueue(messageIds);
        Iterable<List<MessageIdImpl>> batches = Iterables.partition(messageIds.stream().map(messageId -> (MessageIdImpl) messageId).collect(Collectors.toSet()), MAX_REDELIVER_UNACKNOWLEDGED);
        MessageIdData.Builder builder = MessageIdData.newBuilder();
        batches.forEach(ids -> {
            List<MessageIdData> messageIdDatas = ids.stream().map(messageId -> {
                // attempt to remove message from batchMessageAckTracker
                builder.setPartition(messageId.getPartitionIndex());
                builder.setLedgerId(messageId.getLedgerId());
                builder.setEntryId(messageId.getEntryId());
                return builder.build();
            }).collect(Collectors.toList());
            ByteBuf cmd = Commands.newRedeliverUnacknowledgedMessages(consumerId, messageIdDatas);
            cnx.ctx().writeAndFlush(cmd, cnx.ctx().voidPromise());
            messageIdDatas.forEach(MessageIdData::recycle);
        });
        if (messagesFromQueue > 0) {
            increaseAvailablePermits(cnx, messagesFromQueue);
        }
        builder.recycle();
        if (log.isDebugEnabled()) {
            log.debug("[{}] [{}] [{}] Redeliver unacked messages and increase {} permits", subscription, topic, consumerName, messagesFromQueue);
        }
        return;
    }
    if (cnx == null || (getState() == State.Connecting)) {
        log.warn("[{}] Client Connection needs to be establised for redelivery of unacknowledged messages", this);
    } else {
        log.warn("[{}] Reconnecting the client to redeliver the messages.", this);
        cnx.ctx().close();
    }
}

Example 52 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class PersistentFailoverE2ETest method testActiveConsumerFailoverWithDelay.

@Test
public void testActiveConsumerFailoverWithDelay() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/failover-topic3";
    final String subName = "sub1";
    final int numMsgs = 100;
    List<Message<byte[]>> receivedMessages = Lists.newArrayList();
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Failover).messageListener((consumer, msg) -> {
        try {
            synchronized (receivedMessages) {
                receivedMessages.add(msg);
            }
            consumer.acknowledge(msg);
        } catch (Exception e) {
            fail("Should not fail");
        }
    });
    ConsumerBuilder<byte[]> consumerBuilder1 = consumerBuilder.clone().consumerName("1");
    ConsumerBuilder<byte[]> consumerBuilder2 = consumerBuilder.clone().consumerName("2");
    conf.setActiveConsumerFailoverDelayTimeMillis(500);
    restartBroker();
    // create subscription
    Consumer<byte[]> consumer = consumerBuilder1.subscribe();
    consumer.close();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    // enqueue messages
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "my-message-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    futures.clear();
    producer.close();
    // two consumers subscribe at almost the same time
    CompletableFuture<Consumer<byte[]>> subscribeFuture2 = consumerBuilder2.subscribeAsync();
    CompletableFuture<Consumer<byte[]>> subscribeFuture1 = consumerBuilder1.subscribeAsync();
    // wait for all messages to be dequeued
    int retry = 20;
    for (int i = 0; i < retry; i++) {
        if (receivedMessages.size() >= numMsgs && subRef.getNumberOfEntriesInBacklog() == 0) {
            break;
        } else if (i != retry - 1) {
            Thread.sleep(100);
        }
    }
    // check if message duplication has occurred
    assertEquals(receivedMessages.size(), numMsgs);
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    for (int i = 0; i < receivedMessages.size(); i++) {
        Assert.assertNotNull(receivedMessages.get(i));
        Assert.assertEquals(new String(receivedMessages.get(i).getData()), "my-message-" + i);
    }
    subscribeFuture1.get().close();
    subscribeFuture2.get().unsubscribe();
    admin.persistentTopics().delete(topicName);
    resetConfig();
    restartBroker();
}

Example 53 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class PersistentQueueE2ETest method testRoundRobinBatchDistribution.

// this test is good to have to see the distribution, but every now and then it gets slightly different than the
// expected numbers. keeping this disabled to not break the build, but nevertheless this gives good insight into
// how the round robin distribution algorithm is behaving
@Test(enabled = false)
public void testRoundRobinBatchDistribution() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic5";
    final String subName = "sub5";
    final int numMsgs = 137;
    /* some random number different than default batch size of 100 */
    final AtomicInteger counter1 = new AtomicInteger(0);
    final AtomicInteger counter2 = new AtomicInteger(0);
    final AtomicInteger counter3 = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(numMsgs * 3);
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).receiverQueueSize(10).subscriptionType(SubscriptionType.Shared);
    Consumer<byte[]> consumer1 = consumerBuilder.clone().messageListener((consumer, msg) -> {
        try {
            counter1.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    }).subscribe();
    Consumer<byte[]> consumer2 = consumerBuilder.clone().messageListener((consumer, msg) -> {
        try {
            counter2.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    }).subscribe();
    Consumer<byte[]> consumer3 = consumerBuilder.clone().messageListener((consumer, msg) -> {
        try {
            counter1.incrementAndGet();
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (Exception e) {
            fail("Should not fail");
        }
    }).subscribe();
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < numMsgs * 3; i++) {
        String message = "msg-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    producer.close();
    latch.await(1, TimeUnit.SECONDS);
    /*
         * total messages = 137 * 3 = 411 Each consumer has 10 permits. There will be 411 / 3*10 = 13 full distributions
         * i.e. each consumer will get 130 messages. In the 14th round, the balance is 411 - 130*3 = 21. Two consumers
         * will get another batch of 10 messages (Total: 140) and the 3rd one will get the last one (Total: 131)
         */
    assertTrue(CollectionUtils.subtract(Lists.newArrayList(140, 140, 131), Lists.newArrayList(counter1.get(), counter2.get(), counter3.get())).isEmpty());
    consumer1.close();
    consumer2.close();
    consumer3.close();
    admin.persistentTopics().delete(topicName);
}

Example 54 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class PersistentQueueE2ETest method testConsumersWithDifferentPermits.

@Test
public void testConsumersWithDifferentPermits() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/shared-topic4";
    final String subName = "sub4";
    final int numMsgs = 10000;
    final AtomicInteger msgCountConsumer1 = new AtomicInteger(0);
    final AtomicInteger msgCountConsumer2 = new AtomicInteger(0);
    final CountDownLatch latch = new CountDownLatch(numMsgs);
    int recvQ1 = 10;
    Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ1).messageListener((consumer, msg) -> {
        msgCountConsumer1.incrementAndGet();
        try {
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (PulsarClientException e) {
            fail("Should not fail");
        }
    }).subscribe();
    int recvQ2 = 1;
    Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).subscriptionType(SubscriptionType.Shared).receiverQueueSize(recvQ2).messageListener((consumer, msg) -> {
        msgCountConsumer2.incrementAndGet();
        try {
            consumer.acknowledge(msg);
            latch.countDown();
        } catch (PulsarClientException e) {
            fail("Should not fail");
        }
    }).subscribe();
    List<CompletableFuture<MessageId>> futures = Lists.newArrayListWithCapacity(numMsgs);
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).maxPendingMessages(numMsgs + 1).create();
    for (int i = 0; i < numMsgs; i++) {
        String message = "msg-" + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    FutureUtil.waitForAll(futures).get();
    producer.close();
    latch.await(5, TimeUnit.SECONDS);
    assertEquals(msgCountConsumer1.get(), numMsgs - numMsgs / (recvQ1 + recvQ2), numMsgs * 0.1);
    assertEquals(msgCountConsumer2.get(), numMsgs / (recvQ1 + recvQ2), numMsgs * 0.1);
    consumer1.close();
    consumer2.close();
    admin.persistentTopics().delete(topicName);
}

Example 55 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testECDSAEncryption.

@Test(groups = "encryption")
public void testECDSAEncryption() throws Exception {
    log.info("-- Starting {} test --", methodName);
    class EncKeyReader implements CryptoKeyReader {

        EncryptionKeyInfo keyInfo = new EncryptionKeyInfo();

        @Override
        public EncryptionKeyInfo getPublicKey(String keyName, Map<String, String> keyMeta) {
            String CERT_FILE_PATH = "./src/test/resources/certificate/public-key." + keyName;
            if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
                try {
                    keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
                    return keyInfo;
                } catch (IOException e) {
                    Assert.fail("Failed to read certificate from " + CERT_FILE_PATH);
                }
            } else {
                Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable.");
            }
            return null;
        }

        @Override
        public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMeta) {
            String CERT_FILE_PATH = "./src/test/resources/certificate/private-key." + keyName;
            if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
                try {
                    keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
                    return keyInfo;
                } catch (IOException e) {
                    Assert.fail("Failed to read certificate from " + CERT_FILE_PATH);
                }
            } else {
                Assert.fail("Certificate file " + CERT_FILE_PATH + " is not present or not readable.");
            }
            return null;
        }
    }
    final int totalMsg = 10;
    Set<String> messageSet = Sets.newHashSet();
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    conf.setCryptoKeyReader(new EncKeyReader());
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/myecdsa-topic1", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.addEncryptionKey("client-ecdsa.pem");
    producerConf.setCryptoKeyReader(new EncKeyReader());
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/myecdsa-topic1", producerConf);
    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    for (int i = 0; i < totalMsg; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}