Examples with Consumer com.yahoo.pulsar.client.api.Consumer used on opensource projects

Example 41 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class MessageIdTest method partitionedProducerSendAsync.

@Test(timeOut = 10000)
public void partitionedProducerSendAsync() throws PulsarClientException, PulsarAdminException {
    // 1. Basic Config
    String key = "partitionedProducerSendAsync";
    final String topicName = "persistent://prop/cluster/namespace/topic-" + key;
    final String subscriptionName = "my-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int numberOfMessages = 30;
    int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    // 4. Publish message and get message id
    Set<MessageId> messageIds = new HashSet();
    Set<Future<MessageId>> futures = new HashSet();
    for (int i = 0; i < numberOfMessages; i++) {
        String message = messagePredicate + i;
        futures.add(producer.sendAsync(message.getBytes()));
    }
    futures.forEach(f -> {
        try {
            messageIds.add(f.get());
        } catch (Exception e) {
            Assert.fail("Failed to publish message, Exception: " + e.getMessage());
        }
    });
    // 4. Check if message Ids are correct
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), numberOfMessages, "Not all messages published successfully");
    for (int i = 0; i < numberOfMessages; i++) {
        MessageId messageId = consumer.receive().getMessageId();
        log.info("Message ID Received = " + messageId);
        Assert.assertTrue(messageIds.remove(messageId), "Failed to receive Message");
    }
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), 0, "Not all messages received successfully");
    consumer.unsubscribe();
}

Example 42 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class MessageIdTest method partitionedProducerSend.

@Test(timeOut = 10000)
public void partitionedProducerSend() throws PulsarClientException, PulsarAdminException {
    // 1. Basic Config
    String key = "partitionedProducerSend";
    final String topicName = "persistent://prop/cluster/namespace/topic-" + key;
    final String subscriptionName = "my-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int numberOfMessages = 30;
    int numberOfPartitions = 7;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    // 4. Publish message and get message id
    Set<MessageId> messageIds = new HashSet();
    for (int i = 0; i < numberOfMessages; i++) {
        String message = messagePredicate + i;
        messageIds.add(producer.send(message.getBytes()));
    }
    // 4. Check if message Ids are correct
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), numberOfMessages, "Not all messages published successfully");
    for (int i = 0; i < numberOfMessages; i++) {
        Assert.assertTrue(messageIds.remove(consumer.receive().getMessageId()), "Failed to receive Message");
    }
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), 0, "Not all messages received successfully");
// TODO - this statement causes the broker to hang - need to look into
// it
// consumer.unsubscribe();;
}

Example 43 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class PerMessageUnAcknowledgedRedeliveryTest method testExclusiveAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testExclusiveAckedNormalTopic() throws Exception {
    String key = "testExclusiveAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 15;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(50);
    conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String message = messagePredicate + i;
        log.info("Producer produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message, doesn't ack
    Message message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    // 5. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 6. Receiver receives the message, ack them
    message = consumer.receive();
    int received = 0;
    while (message != null) {
        received++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    assertEquals(received, 5);
    // 7. Simulate ackTimeout
    ((ConsumerImpl) consumer).getUnAckedMessageTracker().toggle();
    // 8. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 9. Receiver receives the message, doesn't ack
    message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 10);
    Thread.sleep(ackTimeOutMillis);
    // 10. Receiver receives redelivered messages
    message = consumer.receive();
    int redelivered = 0;
    while (message != null) {
        redelivered++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    assertEquals(redelivered, 10);
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 0);
}

Example 44 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class PerMessageUnAcknowledgedRedeliveryTest method testSharedAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testSharedAckedNormalTopic() throws Exception {
    String key = "testSharedAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 15;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(50);
    conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String message = messagePredicate + i;
        log.info("Producer produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message, doesn't ack
    Message message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    // 5. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 6. Receiver receives the message, ack them
    message = consumer.receive();
    int received = 0;
    while (message != null) {
        received++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    assertEquals(received, 5);
    // 7. Simulate ackTimeout
    ((ConsumerImpl) consumer).getUnAckedMessageTracker().toggle();
    // 8. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 9. Receiver receives the message, doesn't ack
    message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 10);
    Thread.sleep(ackTimeOutMillis);
    // 10. Receiver receives redelivered messages
    message = consumer.receive();
    int redelivered = 0;
    while (message != null) {
        redelivered++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    assertEquals(redelivered, 5);
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
}

Example 45 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class PerMessageUnAcknowledgedRedeliveryTest method testFailoverAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testFailoverAckedNormalTopic() throws Exception {
    String key = "testFailoverAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 15;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(50);
    conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    conf.setSubscriptionType(SubscriptionType.Failover);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String message = messagePredicate + i;
        log.info("Producer produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message, doesn't ack
    Message message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    // 5. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 6. Receiver receives the message, ack them
    message = consumer.receive();
    int received = 0;
    while (message != null) {
        received++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    assertEquals(received, 5);
    // 7. Simulate ackTimeout
    ((ConsumerImpl) consumer).getUnAckedMessageTracker().toggle();
    // 8. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 9. Receiver receives the message, doesn't ack
    message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 10);
    Thread.sleep(ackTimeOutMillis);
    // 10. Receiver receives redelivered messages
    message = consumer.receive();
    int redelivered = 0;
    while (message != null) {
        redelivered++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    assertEquals(redelivered, 10);
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 0);
}