Examples with Consumer com.yahoo.pulsar.client.api.Consumer used on opensource projects

Example 46 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class PerMessageUnAcknowledgedRedeliveryTest method testSharedAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testSharedAckedPartitionedTopic() throws Exception {
    String key = "testSharedAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 15;
    final int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(50);
    conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    conf.setSubscriptionType(SubscriptionType.Shared);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String message = messagePredicate + i;
        log.info("Producer produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message, doesn't ack
    Message message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    long size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    // 5. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 6. Receiver receives the message, ack them
    message = consumer.receive();
    int received = 0;
    while (message != null) {
        received++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
    assertEquals(received, 5);
    // 7. Simulate ackTimeout
    ((PartitionedConsumerImpl) consumer).getConsumers().forEach(c -> c.getUnAckedMessageTracker().toggle());
    // 8. producer publish more messages
    for (int i = 0; i < totalMessages / 3; i++) {
        String m = messagePredicate + i;
        log.info("Producer produced: " + m);
        producer.send(m.getBytes());
    }
    // 9. Receiver receives the message, doesn't ack
    message = consumer.receive();
    while (message != null) {
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 10);
    Thread.sleep(ackTimeOutMillis);
    // 10. Receiver receives redelivered messages
    message = consumer.receive();
    int redelivered = 0;
    while (message != null) {
        redelivered++;
        String data = new String(message.getData());
        log.info("Consumer received : " + data);
        consumer.acknowledge(message);
        message = consumer.receive(100, TimeUnit.MILLISECONDS);
    }
    assertEquals(redelivered, 5);
    size = getUnackedMessagesCountInPartitionedConsumer(consumer);
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 5);
}

Example 47 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class UnAcknowledgedMessagesTimeoutTest method testSharedSingleAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testSharedSingleAckedPartitionedTopic() throws Exception {
    String key = "testSharedSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-shared-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 20;
    final int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // Special step to create partitioned topic
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setReceiverQueueSize(100);
    consumerConfig.setSubscriptionType(SubscriptionType.Shared);
    consumerConfig.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    consumerConfig.setConsumerName("Consumer-1");
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    consumerConfig.setConsumerName("Consumer-2");
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        MessageId msgId = producer.send(message.getBytes());
        log.info("Message produced: {} -- msgId: {}", message, msgId);
    }
    // 4. Receive messages
    int messageCount1 = receiveAllMessage(consumer1, false);
    int messageCount2 = receiveAllMessage(consumer2, true);
    int ackCount1 = 0;
    int ackCount2 = messageCount2;
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    // 5. Check if Messages redelivered again
    // Since receive is a blocking call hoping that timeout will kick in
    Thread.sleep((int) (ackTimeOutMillis * 1.1));
    log.info(key + " Timeout should be triggered now");
    messageCount1 = receiveAllMessage(consumer1, true);
    messageCount2 += receiveAllMessage(consumer2, false);
    ackCount1 = messageCount1;
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    assertEquals(ackCount1 + messageCount2, totalMessages);
    Thread.sleep((int) (ackTimeOutMillis * 1.1));
    // Since receive is a blocking call hoping that timeout will kick in
    log.info(key + " Timeout should be triggered again");
    ackCount1 += receiveAllMessage(consumer1, true);
    ackCount2 += receiveAllMessage(consumer2, true);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(ackCount1 + ackCount2, totalMessages);
}

Example 48 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class UnAcknowledgedMessagesTimeoutTest method testExclusiveCumulativeAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testExclusiveCumulativeAckedNormalTopic() throws Exception {
    String key = "testExclusiveCumulativeAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(7);
    conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message
    HashSet<String> hSet = new HashSet<>();
    Message message = consumer.receive();
    Message lastMessage = message;
    while (message != null) {
        lastMessage = message;
        hSet.add(new String(message.getData()));
        log.info("Consumer received " + new String(message.getData()));
        log.info("Message ID details " + ((MessageIdImpl) message.getMessageId()).toString());
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    }
    long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    assertEquals(size, totalMessages);
    log.info("Comulative Ack sent for " + new String(lastMessage.getData()));
    log.info("Message ID details " + ((MessageIdImpl) lastMessage.getMessageId()).toString());
    consumer.acknowledgeCumulative(lastMessage);
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    assertEquals(size, 0);
    message = consumer.receive((int) (2 * ackTimeOutMillis), TimeUnit.MILLISECONDS);
    assertEquals(message, null);
}

Example 49 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class ClientErrorsTest method testOneConsumerFailShouldCloseAllConsumersInPartitionedConsumer.

// if a consumer fails to subscribe while creating partitioned consumer, it should close all successful connections
// of other consumers and fail
@Test
public void testOneConsumerFailShouldCloseAllConsumersInPartitionedConsumer() throws Exception {
    PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
    final AtomicInteger subscribeCounter = new AtomicInteger(0);
    final AtomicInteger closeCounter = new AtomicInteger(0);
    mockBrokerService.setHandleSubscribe((ctx, subscribe) -> {
        if (subscribeCounter.incrementAndGet() == 3) {
            ctx.writeAndFlush(Commands.newError(subscribe.getRequestId(), ServerError.AuthenticationError, "msg"));
            return;
        }
        ctx.writeAndFlush(Commands.newSuccess(subscribe.getRequestId()));
    });
    mockBrokerService.setHandleCloseConsumer((ctx, closeConsumer) -> {
        ctx.writeAndFlush(Commands.newSuccess(closeConsumer.getRequestId()));
        closeCounter.incrementAndGet();
    });
    try {
        Consumer consumer = client.subscribe("persistent://prop/use/ns/multi-part-t1", "my-sub");
        fail("Should have failed with an authentication error");
    } catch (Exception e) {
        assertTrue(e instanceof PulsarClientException.AuthenticationException);
        // should call close for 3 partitions
        assertEquals(closeCounter.get(), 3);
    }
    mockBrokerService.resetHandleSubscribe();
    mockBrokerService.resetHandleCloseConsumer();
    client.close();
}

Example 50 with Consumer

use of com.yahoo.pulsar.client.api.Consumer in project pulsar by yahoo.

the class ClientErrorsTest method subscribeFailWithoutRetry.

private void subscribeFailWithoutRetry(String topic) throws Exception {
    PulsarClient client = PulsarClient.create("http://127.0.0.1:" + WEB_SERVICE_PORT);
    final AtomicInteger counter = new AtomicInteger(0);
    mockBrokerService.setHandleSubscribe((ctx, subscribe) -> {
        if (counter.incrementAndGet() == 2) {
            // piggyback unknown error to relay assertion failure
            ctx.writeAndFlush(Commands.newError(subscribe.getRequestId(), ServerError.UnknownError, ASSERTION_ERROR));
            return;
        }
        ctx.writeAndFlush(Commands.newError(subscribe.getRequestId(), ServerError.PersistenceError, "msg"));
    });
    try {
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setSubscriptionType(SubscriptionType.Exclusive);
        Consumer consumer = client.subscribe(topic, "sub1", conf);
    } catch (Exception e) {
        if (e.getMessage().equals(ASSERTION_ERROR)) {
            fail("Subscribe should not retry on persistence error");
        }
        assertTrue(e instanceof PulsarClientException.BrokerPersistenceException);
    }
    mockBrokerService.resetHandleSubscribe();
    client.close();
}