Examples with Message org.apache.pulsar.client.api.Message used on opensource projects

Example 1 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class TopicsConsumerImplTest method testAsyncConsumer.

@Test(timeOut = testTimeout)
public void testAsyncConsumer() throws Exception {
    String key = "TopicsConsumerAsyncTest";
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 30;
    final String topicName1 = "persistent://prop/use/ns-abc/topic-1-" + key;
    final String topicName2 = "persistent://prop/use/ns-abc/topic-2-" + key;
    final String topicName3 = "persistent://prop/use/ns-abc/topic-3-" + key;
    List<String> topicNames = Lists.newArrayList(topicName1, topicName2, topicName3);
    admin.properties().createProperty("prop", new PropertyAdmin());
    admin.persistentTopics().createPartitionedTopic(topicName2, 2);
    admin.persistentTopics().createPartitionedTopic(topicName3, 3);
    // 1. producer connect
    Producer<byte[]> producer1 = pulsarClient.newProducer().topic(topicName1).create();
    Producer<byte[]> producer2 = pulsarClient.newProducer().topic(topicName2).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    Producer<byte[]> producer3 = pulsarClient.newProducer().topic(topicName3).messageRoutingMode(org.apache.pulsar.client.api.MessageRoutingMode.RoundRobinPartition).create();
    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topics(topicNames).subscriptionName(subscriptionName).subscriptionType(SubscriptionType.Shared).ackTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS).receiverQueueSize(4).subscribe();
    assertTrue(consumer instanceof TopicsConsumerImpl);
    // Asynchronously produce messages
    List<Future<MessageId>> futures = Lists.newArrayList();
    for (int i = 0; i < totalMessages / 3; i++) {
        futures.add(producer1.sendAsync((messagePredicate + "producer1-" + i).getBytes()));
        futures.add(producer2.sendAsync((messagePredicate + "producer2-" + i).getBytes()));
        futures.add(producer3.sendAsync((messagePredicate + "producer3-" + i).getBytes()));
    }
    log.info("Waiting for async publish to complete : {}", futures.size());
    for (Future<MessageId> future : futures) {
        future.get();
    }
    log.info("start async consume");
    CountDownLatch latch = new CountDownLatch(totalMessages);
    ExecutorService executor = Executors.newFixedThreadPool(1);
    executor.execute(() -> IntStream.range(0, totalMessages).forEach(index -> consumer.receiveAsync().thenAccept(msg -> {
        assertTrue(msg instanceof TopicMessageImpl);
        try {
            consumer.acknowledge(msg);
        } catch (PulsarClientException e1) {
            fail("message acknowledge failed", e1);
        }
        latch.countDown();
        log.info("receive index: {}, latch countDown: {}", index, latch.getCount());
    }).exceptionally(ex -> {
        log.warn("receive index: {}, failed receive message {}", index, ex.getMessage());
        ex.printStackTrace();
        return null;
    })));
    latch.await();
    log.info("success latch wait");
    consumer.unsubscribe();
    consumer.close();
    producer1.close();
    producer2.close();
    producer3.close();
}

Example 2 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class CompactionTest method testFirstMessageRetained.

@Test
public void testFirstMessageRetained() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-topic1";
    // subscribe before sending anything, so that we get all messages
    pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close();
    try (Producer producer = pulsarClient.createProducer(topic)) {
        producer.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-message-1".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-message-2".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-message-3".getBytes()).build()).get();
    }
    // Read messages before compaction to get ids
    List<Message> messages = new ArrayList<>();
    try (Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe()) {
        messages.add(consumer.receive());
        messages.add(consumer.receive());
        messages.add(consumer.receive());
    }
    // compact the topic
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();
    // Check that messages after compaction have same ids
    try (Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe()) {
        Message message1 = consumer.receive();
        Assert.assertEquals(message1.getKey(), "key1");
        Assert.assertEquals(new String(message1.getData()), "my-message-1");
        Assert.assertEquals(message1.getMessageId(), messages.get(0).getMessageId());
        Message message2 = consumer.receive();
        Assert.assertEquals(message2.getKey(), "key2");
        Assert.assertEquals(new String(message2.getData()), "my-message-3");
        Assert.assertEquals(message2.getMessageId(), messages.get(2).getMessageId());
    }
}

Example 3 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class CompactionTest method testBatchMessageIdsDontChange.

@Test
public void testBatchMessageIdsDontChange() throws Exception {
    String topic = "persistent://my-property/use/my-ns/my-topic1";
    // subscribe before sending anything, so that we get all messages
    pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe().close();
    try (Producer producer = pulsarClient.newProducer().topic(topic).maxPendingMessages(3).enableBatching(true).batchingMaxMessages(3).batchingMaxPublishDelay(1, TimeUnit.HOURS).create()) {
        producer.sendAsync(MessageBuilder.create().setKey("key1").setContent("my-message-1".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-message-2".getBytes()).build());
        producer.sendAsync(MessageBuilder.create().setKey("key2").setContent("my-message-3".getBytes()).build()).get();
    }
    // Read messages before compaction to get ids
    List<Message> messages = new ArrayList<>();
    try (Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe()) {
        messages.add(consumer.receive());
        messages.add(consumer.receive());
        messages.add(consumer.receive());
    }
    // Ensure all messages are in same batch
    Assert.assertEquals(((BatchMessageIdImpl) messages.get(0).getMessageId()).getLedgerId(), ((BatchMessageIdImpl) messages.get(1).getMessageId()).getLedgerId());
    Assert.assertEquals(((BatchMessageIdImpl) messages.get(0).getMessageId()).getLedgerId(), ((BatchMessageIdImpl) messages.get(2).getMessageId()).getLedgerId());
    Assert.assertEquals(((BatchMessageIdImpl) messages.get(0).getMessageId()).getEntryId(), ((BatchMessageIdImpl) messages.get(1).getMessageId()).getEntryId());
    Assert.assertEquals(((BatchMessageIdImpl) messages.get(0).getMessageId()).getEntryId(), ((BatchMessageIdImpl) messages.get(2).getMessageId()).getEntryId());
    // compact the topic
    Compactor compactor = new TwoPhaseCompactor(conf, pulsarClient, bk, compactionScheduler);
    compactor.compact(topic).get();
    // Check that messages after compaction have same ids
    try (Consumer consumer = pulsarClient.newConsumer().topic(topic).subscriptionName("sub1").readCompacted(true).subscribe()) {
        Message message1 = consumer.receive();
        Assert.assertEquals(message1.getKey(), "key1");
        Assert.assertEquals(new String(message1.getData()), "my-message-1");
        Assert.assertEquals(message1.getMessageId(), messages.get(0).getMessageId());
        Message message2 = consumer.receive();
        Assert.assertEquals(message2.getKey(), "key2");
        Assert.assertEquals(new String(message2.getData()), "my-message-3");
        Assert.assertEquals(message2.getMessageId(), messages.get(2).getMessageId());
    }
}

Example 4 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class PersistentTopicsImpl method getIndividualMsgsFromBatch.

private List<Message<byte[]>> getIndividualMsgsFromBatch(String msgId, byte[] data, Map<String, String> properties) {
    List<Message<byte[]>> ret = new ArrayList<>();
    int batchSize = Integer.parseInt(properties.get(BATCH_HEADER));
    for (int i = 0; i < batchSize; i++) {
        String batchMsgId = msgId + ":" + i;
        PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata.newBuilder();
        ByteBuf buf = Unpooled.wrappedBuffer(data);
        try {
            ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(buf, singleMessageMetadataBuilder, i, batchSize);
            SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.build();
            if (singleMessageMetadata.getPropertiesCount() > 0) {
                for (KeyValue entry : singleMessageMetadata.getPropertiesList()) {
                    properties.put(entry.getKey(), entry.getValue());
                }
            }
            ret.add(new MessageImpl<>(batchMsgId, properties, singleMessagePayload, Schema.IDENTITY));
        } catch (Exception ex) {
            log.error("Exception occured while trying to get BatchMsgId: {}", batchMsgId, ex);
        }
        buf.release();
        singleMessageMetadataBuilder.recycle();
    }
    return ret;
}

Example 5 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class CmdProduce method run.

/**
 * Run the producer.
 *
 * @return 0 for success, < 0 otherwise
 * @throws Exception
 */
public int run() throws PulsarClientException {
    if (mainOptions.size() != 1)
        throw (new ParameterException("Please provide one and only one topic name."));
    if (this.numTimesProduce <= 0)
        throw (new ParameterException("Number of times need to be positive number."));
    if (messages.size() == 0 && messageFileNames.size() == 0)
        throw (new ParameterException("Please supply message content with either --messages or --files"));
    int totalMessages = (messages.size() + messageFileNames.size()) * numTimesProduce;
    if (totalMessages > MAX_MESSAGES) {
        String msg = "Attempting to send " + totalMessages + " messages. Please do not send more than " + MAX_MESSAGES + " messages";
        throw new ParameterException(msg);
    }
    String topic = this.mainOptions.get(0);
    int numMessagesSent = 0;
    int returnCode = 0;
    try {
        PulsarClient client = clientBuilder.build();
        Producer<byte[]> producer = client.newProducer().topic(topic).create();
        List<byte[]> messageBodies = generateMessageBodies(this.messages, this.messageFileNames);
        RateLimiter limiter = (this.publishRate > 0) ? RateLimiter.create(this.publishRate) : null;
        for (int i = 0; i < this.numTimesProduce; i++) {
            List<Message<byte[]>> messages = generateMessages(messageBodies);
            for (Message<byte[]> msg : messages) {
                if (limiter != null)
                    limiter.acquire();
                producer.send(msg);
                numMessagesSent++;
            }
        }
        client.close();
    } catch (Exception e) {
        LOG.error("Error while producing messages");
        LOG.error(e.getMessage(), e);
        returnCode = -1;
    } finally {
        LOG.info("{} messages successfully produced", numMessagesSent);
    }
    return returnCode;
}