Examples with Message com.yahoo.pulsar.client.api.Message used on opensource projects

Example 76 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class BatchMessageTest method testBatchProducerWithLargeMessage.

@Test(dataProvider = "codec")
public void testBatchProducerWithLargeMessage(CompressionType compressionType) throws Exception {
    int numMsgs = 50;
    int numMsgsInBatch = numMsgs / 2;
    final String topicName = "persistent://prop/use/finance/testBatchProducerWithLargeMessage";
    final String subscriptionName = "large-message-sub-1" + compressionType.toString();
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    consumer.close();
    ProducerConfiguration producerConf = new ProducerConfiguration();
    producerConf.setCompressionType(compressionType);
    producerConf.setBatchingMaxPublishDelay(5000, TimeUnit.MILLISECONDS);
    producerConf.setBatchingMaxMessages(numMsgsInBatch);
    producerConf.setBatchingEnabled(true);
    Producer producer = pulsarClient.createProducer(topicName, producerConf);
    List<CompletableFuture<MessageId>> sendFutureList = Lists.newArrayList();
    for (int i = 0; i < numMsgs; i++) {
        if (i == 25) {
            // send a large message
            byte[] largeMessage = new byte[128 * 1024 + 4];
            Message msg = MessageBuilder.create().setContent(largeMessage).build();
            sendFutureList.add(producer.sendAsync(msg));
        } else {
            byte[] message = ("msg-" + i).getBytes();
            Message msg = MessageBuilder.create().setContent(message).build();
            sendFutureList.add(producer.sendAsync(msg));
        }
    }
    byte[] message = ("msg-" + "last").getBytes();
    Message lastMsg = MessageBuilder.create().setContent(message).build();
    sendFutureList.add(producer.sendAsync(lastMsg));
    FutureUtil.waitForAll(sendFutureList).get();
    PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    rolloverPerIntervalStats();
    assertTrue(topic.getProducers().values().iterator().next().getStats().msgRateIn > 0.0);
    // we expect 3 messages in the backlog since the large message in the middle should
    // close out the batch and be sent in a batch of its own
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 3);
    consumer = pulsarClient.subscribe(topicName, subscriptionName);
    for (int i = 0; i <= numMsgs; i++) {
        Message msg = consumer.receive(5, TimeUnit.SECONDS);
        assertNotNull(msg);
        LOG.info("received msg - {}", msg.getData().toString());
        consumer.acknowledge(msg);
    }
    Thread.sleep(100);
    assertEquals(topic.getPersistentSubscription(subscriptionName).getNumberOfEntriesInBacklog(), 0);
    consumer.close();
    producer.close();
}

Example 77 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class ResendRequestTest method testExclusiveSingleAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testExclusiveSingleAckedPartitionedTopic() throws Exception {
    String key = "testExclusiveSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    final int numberOfPartitions = 4;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // Special step to create partitioned topic
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setReceiverQueueSize(7);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Message produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receive messages
    Message message = consumer.receive();
    int messageCount = 0;
    log.info("Message received " + new String(message.getData()));
    do {
        messageCount += 1;
        log.info("Message received " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageCount, totalMessages);
    // 5. Ask for redeliver
    consumer.redeliverUnacknowledgedMessages();
    // 6. Check if Messages redelivered again
    message = consumer.receive();
    messageCount = 0;
    log.info("Message received " + new String(message.getData()));
    do {
        messageCount += 1;
        log.info("Message received " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(messageCount, totalMessages);
}

Example 78 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class ResendRequestTest method testFailoverSingleAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testFailoverSingleAckedNormalTopic() throws Exception {
    String key = "testFailoverSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(10);
    conf.setSubscriptionType(SubscriptionType.Failover);
    conf.setConsumerName("consumer-1");
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    conf.setConsumerName("consumer-2");
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. Producer publishes messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
        log.info("Producer produced " + message);
    }
    // 4. Receive messages
    int receivedConsumer1 = 0, receivedConsumer2 = 0;
    Message message1;
    Message message2;
    do {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
    } while (message1 != null || message2 != null);
    log.info("Consumer 1 receives = " + receivedConsumer1);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    log.info("Total receives = " + (receivedConsumer2 + receivedConsumer1));
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);
    // 5. Consumer 1 asks for resend
    consumer1.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);
    // 6. Consumer 1 acknowledges a few messages
    receivedConsumer1 = receivedConsumer2 = 0;
    for (int i = 0; i < totalMessages / 2; i++) {
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
        if (message1 != null) {
            log.info("Consumer 1 Received: " + new String(message1.getData()));
            receivedConsumer1 += 1;
            log.info("Consumer 1 Acknowledged: " + new String(message1.getData()));
            consumer1.acknowledge(message1);
        }
        if (message2 != null) {
            log.info("Consumer 2 Received: " + new String(message2.getData()));
            receivedConsumer2 += 1;
        }
    }
    assertEquals(receivedConsumer2 + receivedConsumer1, totalMessages / 2);
    // Consumer 2 is on Stand By
    assertEquals(receivedConsumer2, 0);
    // 7. Consumer 1 close
    consumer1.close();
    // 8. Checking if all messages are received by Consumer 2
    message2 = consumer2.receive();
    int acknowledgedMessages = 0;
    int unAcknowledgedMessages = 0;
    boolean flag = true;
    do {
        if (flag) {
            consumer2.acknowledge(message2);
            acknowledgedMessages += 1;
            log.info("Consumer 2 Acknowledged: " + new String(message2.getData()));
        } else {
            unAcknowledgedMessages += 1;
        }
        flag = !flag;
        log.info("Consumer 2 Received: " + new String(message2.getData()));
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message2 != null);
    log.info("Consumer 2 receives = " + (unAcknowledgedMessages + acknowledgedMessages));
    log.info("Consumer 2 acknowledges = " + acknowledgedMessages);
    assertEquals(unAcknowledgedMessages + acknowledgedMessages, totalMessages - receivedConsumer1);
    // 9 .Consumer 2 asks for a resend
    consumer2.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);
    message2 = consumer2.receive();
    receivedConsumer2 = 0;
    do {
        receivedConsumer2 += 1;
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message2 != null);
    log.info("Consumer 2 receives = " + receivedConsumer2);
    assertEquals(unAcknowledgedMessages, receivedConsumer2);
}

Example 79 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class ResendRequestTest method testSharedSingleAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testSharedSingleAckedPartitionedTopic() throws Exception {
    String key = "testSharedSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    final int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    Random rn = new Random();
    // Special step to create partitioned topic
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setReceiverQueueSize(7);
    consumerConfig.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Message produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receive messages
    Message message1 = consumer1.receive();
    Message message2 = consumer2.receive();
    int messageCount1 = 0;
    int messageCount2 = 0;
    int ackCount1 = 0;
    int ackCount2 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer1.acknowledge(message1);
                log.info("Consumer1 acked " + new String(message1.getData()));
                ackCount1 += 1;
            }
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer2.acknowledge(message2);
                log.info("Consumer2 acked " + new String(message2.getData()));
                ackCount2 += 1;
            }
        }
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    // 5. Ask for redeliver
    log.info(key + ": Sent a Redeliver Message Request");
    consumer1.redeliverUnacknowledgedMessages();
    if ((ackCount1 + ackCount2) == totalMessages) {
        return;
    }
    // 6. Check if Messages redelivered again
    message1 = consumer1.receive(5000, TimeUnit.MILLISECONDS);
    message2 = consumer2.receive(5000, TimeUnit.MILLISECONDS);
    messageCount1 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
        }
        message1 = consumer1.receive(1000, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(1000, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2 + ackCount1, totalMessages);
}

Example 80 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class ResendRequestTest method testFailoverSingleAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testFailoverSingleAckedPartitionedTopic() throws Exception {
    String key = "testFailoverSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    final int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    Random rn = new Random();
    // Special step to create partitioned topic
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setReceiverQueueSize(7);
    consumerConfig.setSubscriptionType(SubscriptionType.Failover);
    consumerConfig.setConsumerName("Consumer-1");
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    consumerConfig.setConsumerName("Consumer-2");
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    Thread.sleep(1000);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Message produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receive messages
    Message message1 = consumer1.receive();
    Message message2 = consumer2.receive();
    int messageCount1 = 0;
    int messageCount2 = 0;
    int ackCount1 = 0;
    int ackCount2 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer1.acknowledge(message1);
                ackCount1 += 1;
            }
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
            if (rn.nextInt() % 3 == 0) {
                consumer2.acknowledge(message2);
                ackCount2 += 1;
            }
        }
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    if ((ackCount1 + ackCount2) == totalMessages) {
        return;
    }
    // 5. Ask for redeliver
    log.info(key + ": Sent a Redeliver Message Request");
    consumer1.redeliverUnacknowledgedMessages();
    consumer1.close();
    // 6. Check if Messages redelivered again
    message2 = consumer2.receive();
    messageCount1 = 0;
    do {
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
        }
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount2 + ackCount1, totalMessages);
}