Examples with MessageId com.yahoo.pulsar.client.api.MessageId used on opensource projects

Example 1 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class ResendRequestTest method testExclusiveSingleAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testExclusiveSingleAckedNormalTopic() throws Exception {
    String key = "testExclusiveSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    HashSet<MessageId> messageIdHashSet = new HashSet<MessageId>();
    HashSet<String> messageDataHashSet = new HashSet<String>();
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(7);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
    }
    // 4. Receive messages
    Message message = consumer.receive();
    log.info("Message received " + new String(message.getData()));
    for (int i = 1; i < totalMessages; i++) {
        Message msg = consumer.receive();
        log.info("Message received " + new String(msg.getData()));
        messageDataHashSet.add(new String(msg.getData()));
    }
    printIncomingMessageQueue(consumer);
    // 5. Ack 1st message and ask for resend
    consumer.acknowledge(message);
    log.info("Message acked " + new String(message.getData()));
    messageIdHashSet.add(message.getMessageId());
    messageDataHashSet.add(new String(message.getData()));
    consumer.redeliverUnacknowledgedMessages();
    log.info("Resend Messages Request sent");
    // 6. Check if messages resent in correct order
    for (int i = 0; i < totalMessages - 1; i++) {
        message = consumer.receive();
        log.info("Message received " + new String(message.getData()));
        if (i < 2) {
            messageIdHashSet.add(message.getMessageId());
            consumer.acknowledge(message);
        }
        log.info("Message acked " + new String(message.getData()));
        assertTrue(messageDataHashSet.contains(new String(message.getData())));
    }
    assertEquals(messageIdHashSet.size(), 3);
    assertEquals(messageDataHashSet.size(), totalMessages);
    printIncomingMessageQueue(consumer);
    // 7. Request resend 2nd time - you should receive 4 messages
    consumer.redeliverUnacknowledgedMessages();
    log.info("Resend Messages Request sent");
    message = consumer.receive(2000, TimeUnit.MILLISECONDS);
    while (message != null) {
        log.info("Message received " + new String(message.getData()));
        consumer.acknowledge(message);
        log.info("Message acked " + new String(message.getData()));
        messageIdHashSet.add(message.getMessageId());
        messageDataHashSet.add(new String(message.getData()));
        message = consumer.receive(5000, TimeUnit.MILLISECONDS);
    }
    assertEquals(messageIdHashSet.size(), totalMessages);
    assertEquals(messageDataHashSet.size(), totalMessages);
    printIncomingMessageQueue(consumer);
    // 9. Calling resend after acking all messages - expectin 0 messages
    consumer.redeliverUnacknowledgedMessages();
    assertEquals(consumer.receive(2000, TimeUnit.MILLISECONDS), null);
    // 10. Checking message contents
    for (int i = 0; i < totalMessages; i++) {
        assertTrue(messageDataHashSet.contains(messagePredicate + i));
    }
}

Example 2 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class MessageIdTest method testChecksumReconnection.

@Test
public void testChecksumReconnection() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic1";
    // 1. producer connect
    ProducerImpl prod = (ProducerImpl) pulsarClient.createProducer(topicName);
    ProducerImpl producer = spy(prod);
    // mock: broker-doesn't support checksum (remote_version < brokerChecksumSupportedVersion) so, it forces
    // client-producer to perform checksum-strip from msg at reconnection
    doReturn(producer.brokerChecksumSupportedVersion() + 1).when(producer).brokerChecksumSupportedVersion();
    doAnswer(invocationOnMock -> prod.getState()).when(producer).getState();
    doAnswer(invocationOnMock -> prod.getClientCnx()).when(producer).getClientCnx();
    doAnswer(invocationOnMock -> prod.cnx()).when(producer).cnx();
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub");
    stopBroker();
    // stop timer to auto-reconnect as let spy-Producer connect to broker
    // manually so, spy-producer object can get
    // mock-value from brokerChecksumSupportedVersion
    ((PulsarClientImpl) pulsarClient).timer().stop();
    // set clientCnx mock to get non-checksum supported version
    ClientCnx mockClientCnx = spy(new ClientCnx((PulsarClientImpl) pulsarClient));
    doReturn(producer.brokerChecksumSupportedVersion() - 1).when(mockClientCnx).getRemoteEndpointProtocolVersion();
    prod.setClientCnx(mockClientCnx);
    Message msg1 = MessageBuilder.create().setContent("message-1".getBytes()).build();
    CompletableFuture<MessageId> future1 = producer.sendAsync(msg1);
    Message msg2 = MessageBuilder.create().setContent("message-2".getBytes()).build();
    CompletableFuture<MessageId> future2 = producer.sendAsync(msg2);
    // corrupt the message
    // new content would be
    msg2.getData()[msg2.getData().length - 1] = '3';
    // 'message-3'
    // unset mock
    prod.setClientCnx(null);
    // Restart the broker to have the messages published
    startBroker();
    // grab broker connection with mocked producer which has higher version
    // compare to broker
    prod.grabCnx();
    try {
        // it should not fail: as due to unsupported version of broker:
        // client removes checksum and broker should
        // ignore the checksum validation
        future1.get(10, TimeUnit.SECONDS);
        future2.get(10, TimeUnit.SECONDS);
    } catch (Exception e) {
        e.printStackTrace();
        fail("Broker shouldn't verify checksum for corrupted message and it shouldn't fail");
    }
    ((ConsumerImpl) consumer).grabCnx();
    // We should only receive msg1
    Message msg = consumer.receive(1, TimeUnit.SECONDS);
    assertEquals(new String(msg.getData()), "message-1");
    msg = consumer.receive(1, TimeUnit.SECONDS);
    assertEquals(new String(msg.getData()), "message-3");
}

Example 3 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class MessageIdTest method producerSendAsync.

@Test(timeOut = 10000)
public void producerSendAsync() throws PulsarClientException {
    // 1. Basic Config
    String key = "producerSendAsync";
    final String topicName = "persistent://prop/cluster/namespace/topic-" + key;
    final String subscriptionName = "my-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int numberOfMessages = 30;
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    // 4. Publish message and get message id
    Set<MessageId> messageIds = new HashSet();
    List<Future<MessageId>> futures = new ArrayList();
    for (int i = 0; i < numberOfMessages; i++) {
        String message = messagePredicate + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    MessageIdImpl previousMessageId = null;
    for (Future<MessageId> f : futures) {
        try {
            MessageIdImpl currentMessageId = (MessageIdImpl) f.get();
            if (previousMessageId != null) {
                Assert.assertTrue(currentMessageId.compareTo(previousMessageId) > 0, "Message Ids should be in ascending order");
            }
            messageIds.add(currentMessageId);
            previousMessageId = currentMessageId;
        } catch (Exception e) {
            Assert.fail("Failed to publish message, Exception: " + e.getMessage());
        }
    }
    // 4. Check if message Ids are correct
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), numberOfMessages, "Not all messages published successfully");
    for (int i = 0; i < numberOfMessages; i++) {
        Message message = consumer.receive();
        Assert.assertEquals(new String(message.getData()), messagePredicate + i);
        MessageId messageId = message.getMessageId();
        Assert.assertTrue(messageIds.remove(messageId), "Failed to receive message");
    }
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), 0, "Not all messages received successfully");
    consumer.unsubscribe();
}

Example 4 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class MessageIdTest method partitionedProducerSendAsync.

@Test(timeOut = 10000)
public void partitionedProducerSendAsync() throws PulsarClientException, PulsarAdminException {
    // 1. Basic Config
    String key = "partitionedProducerSendAsync";
    final String topicName = "persistent://prop/cluster/namespace/topic-" + key;
    final String subscriptionName = "my-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int numberOfMessages = 30;
    int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    // 4. Publish message and get message id
    Set<MessageId> messageIds = new HashSet();
    Set<Future<MessageId>> futures = new HashSet();
    for (int i = 0; i < numberOfMessages; i++) {
        String message = messagePredicate + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    futures.forEach(f -> {
        try {
            messageIds.add(f.get());
        } catch (Exception e) {
            Assert.fail("Failed to publish message, Exception: " + e.getMessage());
        }
    });
    // 4. Check if message Ids are correct
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), numberOfMessages, "Not all messages published successfully");
    for (int i = 0; i < numberOfMessages; i++) {
        MessageId messageId = consumer.receive().getMessageId();
        log.info("Message ID Received = " + messageId);
        Assert.assertTrue(messageIds.remove(messageId), "Failed to receive Message");
    }
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), 0, "Not all messages received successfully");
    consumer.unsubscribe();
}

Example 5 with MessageId

use of com.yahoo.pulsar.client.api.MessageId in project pulsar by yahoo.

the class MessageIdTest method partitionedProducerSend.

@Test(timeOut = 10000)
public void partitionedProducerSend() throws PulsarClientException, PulsarAdminException {
    // 1. Basic Config
    String key = "partitionedProducerSend";
    final String topicName = "persistent://prop/cluster/namespace/topic-" + key;
    final String subscriptionName = "my-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int numberOfMessages = 30;
    int numberOfPartitions = 7;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    // 4. Publish message and get message id
    Set<MessageId> messageIds = new HashSet();
    for (int i = 0; i < numberOfMessages; i++) {
        String message = messagePredicate + i;
        messageIds.add(producer.send(message.getBytes()));
    }
    // 4. Check if message Ids are correct
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), numberOfMessages, "Not all messages published successfully");
    for (int i = 0; i < numberOfMessages; i++) {
        Assert.assertTrue(messageIds.remove(consumer.receive().getMessageId()), "Failed to receive Message");
    }
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), 0, "Not all messages received successfully");
// TODO - this statement causes the broker to hang - need to look into
// it
// consumer.unsubscribe();;
}