Examples with MessageId com.yahoo.pulsar.client.api.MessageId used on opensource projects

Example 6 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class UnAcknowledgedMessagesTimeoutTest method testSharedSingleAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testSharedSingleAckedPartitionedTopic() throws Exception {
    String key = "testSharedSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 20;
    final int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // Special step to create partitioned topic
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setReceiverQueueSize(100);
    consumerConfig.setSubscriptionType(SubscriptionType.Shared);
    consumerConfig.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    consumerConfig.setConsumerName("Consumer-1");
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    consumerConfig.setConsumerName("Consumer-2");
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        MessageId msgId = producer.send(message.getBytes());
        log.info("Message produced: {} -- msgId: {}", message, msgId);
    }
    // 4. Receive messages
    int messageCount1 = receiveAllMessage(consumer1, false);
    int messageCount2 = receiveAllMessage(consumer2, true);
    int ackCount1 = 0;
    int ackCount2 = messageCount2;
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    // 5. Check if Messages redelivered again
    // Since receive is a blocking call hoping that timeout will kick in
    Thread.sleep((int) (ackTimeOutMillis * 1.1));
    log.info(key + " Timeout should be triggered now");
    messageCount1 = receiveAllMessage(consumer1, true);
    messageCount2 += receiveAllMessage(consumer2, false);
    ackCount1 = messageCount1;
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    assertEquals(ackCount1 + messageCount2, totalMessages);
    Thread.sleep((int) (ackTimeOutMillis * 1.1));
    // Since receive is a blocking call hoping that timeout will kick in
    log.info(key + " Timeout should be triggered again");
    ackCount1 += receiveAllMessage(consumer1, true);
    ackCount2 += receiveAllMessage(consumer2, true);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(ackCount1 + ackCount2, totalMessages);
}

Example 7 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class ConsumerHandler method onWebSocketText.

@Override
public void onWebSocketText(String message) {
    super.onWebSocketText(message);
    // We should have received an ack
    MessageId msgId;
    try {
        ConsumerAck ack = ObjectMapperFactory.getThreadLocal().readValue(message, ConsumerAck.class);
        msgId = MessageId.fromByteArray(Base64.getDecoder().decode(ack.messageId));
    } catch (IOException e) {
        log.warn("Failed to deserialize message id: {}", message, e);
        close(WebSocketError.FailedToDeserializeFromJSON);
        return;
    }
    consumer.acknowledgeAsync(msgId);
    int pending = pendingMessages.getAndDecrement();
    if (pending >= maxPendingMessages) {
        // Resume delivery
        receiveMessage();
    }
}

Example 8 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class BatchMessageTest method testSimpleBatchProducerConsumer1kMessages.

@Test
public void testSimpleBatchProducerConsumer1kMessages() throws Exception {
    int numMsgs = 2000;
    int numMsgsInBatch = 4;
    final String topicName = "persistent://prop/use/ns-abc/testSimpleBatchProducerConsumer1kMessages";
    final String subscriptionName = "pc1k-sub-1";
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setBatchingMaxPublishDelay(30000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        byte[] message = ("msg-" + i).getBytes();
        Message msg = MessageBuilder.create().setContent(message).build();
        sendFutureList.add(producer.sendAsync(msg));
    }
    FutureUtil.waitForAll(sendFutureList).get();
    int sendError = 0;
    for (CompletableFuture<MessageId> sendFuture : sendFutureList) {
        if (sendFuture.isCompletedExceptionally()) {
            ++sendError;
        }
    }
    if (sendError != 0) {
        LOG.warn("[{}] Error sending {} messages", subscriptionName, sendError);
        numMsgs = numMsgs - sendError;
    }
    LOG.info("[{}] sent {} messages", subscriptionName, numMsgs);
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    // allow stats to be updated..
    Thread.sleep(5000);
    LOG.info("[{}] checking backlog stats..");
    rolloverPerIntervalStats();
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), numMsgs / numMsgsInBatch);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    Message lastunackedMsg = null;
    for (int i = 0; i < numMsgs; i++) {
        Message msg = consumer.receive(1, TimeUnit.SECONDS);
        assertNotNull(msg);
        lastunackedMsg = msg;
    }
    if (lastunackedMsg != null) {
        consumer.acknowledgeCumulative(lastunackedMsg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
}

Example 9 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class PersistentTopicE2ETest method testPayloadCorruptionDetection.

@Test
public void testPayloadCorruptionDetection() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic1";
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
    Message msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
    CompletableFuture<MessageId> future1 = producer.sendAsync(msg1);
    // Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're checksums
    // would have already been computed. If we change the message content at that point, it should result in a
    // checksum validation error
    stopBroker();
    Message msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
    CompletableFuture<MessageId> future2 = producer.sendAsync(msg2);
    // Taint msg2
    // new content would be 'message-3'
    msg2.getData()[msg2.getData().length - 1] = '3';
    // Restart the broker to have the messages published
    startBroker();
    future1.get();
    try {
        future2.get();
        fail("since we corrupted the message, it should be rejected by the broker");
    } catch (Exception e) {
    // ok
    }
    // We should only receive msg1
    Message msg = consumer.receive(1, TimeUnit.SECONDS);
    assertEquals(new String(msg.getData()), "message-1");
    while ((msg = consumer.receive(1, TimeUnit.SECONDS)) != null) {
        assertEquals(new String(msg.getData()), "message-1");
    }
}

Example 10 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class MessageIdSerialization method testProtobufSerialization1.

@Test
void testProtobufSerialization1() throws Exception {
    MessageId id = new MessageIdImpl(1, 2, 3);
    byte[] serializedId = id.toByteArray();
    assertEquals(MessageId.fromByteArray(serializedId), id);
}