Examples with ConsumerConfiguration org.apache.pulsar.client.api.ConsumerConfiguration used on opensource projects

Example 6 with ConsumerConfiguration

use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testConsumerBlockingWithUnAckedMessages.

/**
 * Verify: Consumer stops receiving msg when reach unack-msg limit and starts receiving once acks messages 1.
 * Produce X (600) messages 2. Consumer has receive size (10) and receive message without acknowledging 3. Consumer
 * will stop receiving message after unAckThreshold = 500 4. Consumer acks messages and starts consuming remanining
 * messages This testcase enables checksum sending while producing message and broker verifies the checksum for the
 * message.
 *
 * @throws Exception
 */
@Test
public void testConsumerBlockingWithUnAckedMessages() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 500;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 600;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
        assertEquals(messages.size(), unAckedMessagesBufferSize);
        // start acknowledging messages
        messages.forEach(m -> {
            try {
                consumer.acknowledge(m);
            } catch (PulsarClientException e) {
                fail("ack failed", e);
            }
        });
        // try to consume remaining messages
        int remainingMessages = totalProducedMsgs - messages.size();
        for (int i = 0; i < remainingMessages; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            }
        }
        // total received-messages should match to produced messages
        assertEquals(totalProducedMsgs, messages.size());
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 7 with ConsumerConfiguration

use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testRedeliveryFailOverConsumer.

@Test
public void testRedeliveryFailOverConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int receiverQueueSize = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(receiverQueueSize);
    conf.setSubscriptionType(SubscriptionType.Failover);
    // Only subscribe consumer
    ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
    // (1) First round to produce-consume messages
    int consumeMsgInParts = 4;
    for (int i = 0; i < receiverQueueSize; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        Thread.sleep(10);
    }
    // (1.a) consume first consumeMsgInParts msgs and trigger redeliver
    Message msg = null;
    List<Message> messages1 = Lists.newArrayList();
    for (int i = 0; i < consumeMsgInParts; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg != null) {
            messages1.add(msg);
            consumer.acknowledge(msg);
            log.info("Received message: " + new String(msg.getData()));
        } else {
            break;
        }
    }
    assertEquals(messages1.size(), consumeMsgInParts);
    consumer.redeliverUnacknowledgedMessages();
    // (1.b) consume second consumeMsgInParts msgs and trigger redeliver
    messages1.clear();
    for (int i = 0; i < consumeMsgInParts; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg != null) {
            messages1.add(msg);
            consumer.acknowledge(msg);
            log.info("Received message: " + new String(msg.getData()));
        } else {
            break;
        }
    }
    assertEquals(messages1.size(), consumeMsgInParts);
    consumer.redeliverUnacknowledgedMessages();
    // (2) Second round to produce-consume messages
    for (int i = 0; i < receiverQueueSize; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
        Thread.sleep(100);
    }
    int remainingMsgs = (2 * receiverQueueSize) - (2 * consumeMsgInParts);
    messages1.clear();
    for (int i = 0; i < remainingMsgs; i++) {
        msg = consumer.receive(1, TimeUnit.SECONDS);
        if (msg != null) {
            messages1.add(msg);
            consumer.acknowledge(msg);
            log.info("Received message: " + new String(msg.getData()));
        } else {
            break;
        }
    }
    assertEquals(messages1.size(), remainingMsgs);
    producer.close();
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 8 with ConsumerConfiguration

use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testUnackedBlockAtBatch.

@Test(dataProvider = "batch")
public void testUnackedBlockAtBatch(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int maxUnackedMessages = 20;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 100;
        int totalReceiveMessages = 0;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(maxUnackedMessages);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        if (batchMessageDelayMs != 0) {
            producerConf.setBatchingEnabled(true);
            producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
            producerConf.setBatchingMaxMessages(5);
        }
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        List<CompletableFuture<MessageId>> futures = Lists.newArrayList();
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            futures.add(producer.sendAsync(message.getBytes()));
        }
        FutureUtil.waitForAll(futures).get();
        // (2) Consumer1: consume without ack:
        // try to consume messages: but will be able to consume number of messages = maxUnackedMessages
        Message msg = null;
        List<Message> messages = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer1.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                totalReceiveMessages++;
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // should be blocked due to unack-msgs and should not consume all msgs
        assertNotEquals(messages.size(), totalProducedMsgs);
        // ack for all maxUnackedMessages
        messages.forEach(m -> {
            try {
                consumer1.acknowledge(m);
            } catch (PulsarClientException e) {
                fail("shouldn't have failed ", e);
            }
        });
        // (3) Consumer consumes and ack: so it should consume all remaining messages
        messages.clear();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer1.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages.add(msg);
                totalReceiveMessages++;
                consumer1.acknowledge(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // verify total-consumer messages = total-produce messages
        assertEquals(totalProducedMsgs, totalReceiveMessages);
        producer.close();
        consumer1.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 9 with ConsumerConfiguration

use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testEncryptionFailure.

@Test(groups = "encryption")
public void testEncryptionFailure() throws Exception {
    log.info("-- Starting {} test --", methodName);
    class EncKeyReader implements CryptoKeyReader {

        EncryptionKeyInfo keyInfo = new EncryptionKeyInfo();

        @Override
        public EncryptionKeyInfo getPublicKey(String keyName, Map<String, String> keyMeta) {
            String CERT_FILE_PATH = "./src/test/resources/certificate/public-key." + keyName;
            if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
                try {
                    keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
                    return keyInfo;
                } catch (IOException e) {
                    log.error("Failed to read certificate from {}", CERT_FILE_PATH);
                }
            }
            return null;
        }

        @Override
        public EncryptionKeyInfo getPrivateKey(String keyName, Map<String, String> keyMeta) {
            String CERT_FILE_PATH = "./src/test/resources/certificate/private-key." + keyName;
            if (Files.isReadable(Paths.get(CERT_FILE_PATH))) {
                try {
                    keyInfo.setKey(Files.readAllBytes(Paths.get(CERT_FILE_PATH)));
                    return keyInfo;
                } catch (IOException e) {
                    log.error("Failed to read certificate from {}", CERT_FILE_PATH);
                }
            }
            return null;
        }
    }
    final int totalMsg = 10;
    ProducerConfiguration producerConf = new ProducerConfiguration();
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/myenc-topic1", "my-subscriber-name", conf);
    // 1. Invalid key name
    producerConf.addEncryptionKey("client-non-existant-rsa.pem");
    producerConf.setCryptoKeyReader(new EncKeyReader());
    try {
        pulsarClient.createProducer("persistent://my-property/use/myenc-ns/myenc-topic1", producerConf);
        Assert.fail("Producer creation should not suceed if failing to read key");
    } catch (Exception e) {
    // ok
    }
    // 2. Producer with valid key name
    producerConf = new ProducerConfiguration();
    producerConf.setCryptoKeyReader(new EncKeyReader());
    producerConf.addEncryptionKey("client-rsa.pem");
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/myenc-topic1", producerConf);
    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    // 3. KeyReder is not set by consumer
    // Receive should fail since key reader is not setup
    msg = consumer.receive(5, TimeUnit.SECONDS);
    Assert.assertNull(msg, "Receive should have failed with no keyreader");
    // 4. Set consumer config to consume even if decryption fails
    conf.setCryptoFailureAction(ConsumerCryptoFailureAction.CONSUME);
    consumer.close();
    consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/myenc-topic1", "my-subscriber-name", conf);
    int msgNum = 0;
    try {
        // Receive should proceed and deliver encrypted message
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        String expectedMessage = "my-message-" + msgNum++;
        Assert.assertNotEquals(receivedMessage, expectedMessage, "Received encrypted message " + receivedMessage + " should not match the expected message " + expectedMessage);
        consumer.acknowledgeCumulative(msg);
    } catch (Exception e) {
        Assert.fail("Failed to receive message even aftet ConsumerCryptoFailureAction.CONSUME is set.");
    }
    // 5. Set keyreader and failure action
    conf.setCryptoFailureAction(ConsumerCryptoFailureAction.FAIL);
    consumer.close();
    // Set keyreader
    conf.setCryptoKeyReader(new EncKeyReader());
    consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/myenc-topic1", "my-subscriber-name", conf);
    for (int i = msgNum; i < totalMsg - 1; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    // 6. Set consumer config to discard if decryption fails
    consumer.close();
    ConsumerConfiguration conf2 = new ConsumerConfiguration();
    conf2.setSubscriptionType(SubscriptionType.Exclusive);
    conf2.setCryptoFailureAction(ConsumerCryptoFailureAction.DISCARD);
    consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/myenc-topic1", "my-subscriber-name", conf2);
    // Receive should proceed and discard encrypted messages
    msg = consumer.receive(5, TimeUnit.SECONDS);
    Assert.assertNull(msg, "Message received even aftet ConsumerCryptoFailureAction.DISCARD is set.");
    log.info("-- Exiting {} test --", methodName);
}

Example 10 with ConsumerConfiguration

use of org.apache.pulsar.client.api.ConsumerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testBlockUnackedConsumerRedeliverySpecificMessagesCloseConsumerWhileProduce.

/**
 * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
 * blocked due to unacked messsages
 *
 * Usecase: Consumer starts consuming only after all messages have been produced. So, consumer consumes total
 * receiver-queue-size number messages => ask for redelivery and receives all messages again.
 *
 * @throws Exception
 */
@Test
public void testBlockUnackedConsumerRedeliverySpecificMessagesCloseConsumerWhileProduce() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 10;
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 50;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        // Only subscribe consumer
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        consumer.close();
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
            Thread.sleep(10);
        }
        // (1.a) start consumer again
        consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages1 = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages1.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client should not receive all produced messages and should be blocked due to unack-messages
        assertEquals(messages1.size(), receiverQueueSize);
        Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
            return (MessageIdImpl) m.getMessageId();
        }).collect(Collectors.toSet());
        // (3) redeliver all consumed messages
        consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
        Thread.sleep(1000);
        Set<MessageIdImpl> messages2 = Sets.newHashSet();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages2.add((MessageIdImpl) msg.getMessageId());
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        assertEquals(messages1.size(), messages2.size());
        // (4) Verify: redelivered all previous unacked-consumed messages
        messages2.removeAll(redeliveryMessages);
        assertEquals(messages2.size(), 0);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}