Examples with ProducerConfiguration org.apache.pulsar.client.api.ProducerConfiguration used on opensource projects

Example 11 with ProducerConfiguration

use of org.apache.pulsar.client.api.ProducerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testBlockUnackedConsumerRedeliverySpecificMessagesCloseConsumerWhileProduce.

/**
 * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
 * blocked due to unacked messsages
 *
 * Usecase: Consumer starts consuming only after all messages have been produced. So, consumer consumes total
 * receiver-queue-size number messages => ask for redelivery and receives all messages again.
 *
 * @throws Exception
 */
@Test
public void testBlockUnackedConsumerRedeliverySpecificMessagesCloseConsumerWhileProduce() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 10;
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 50;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        // Only subscribe consumer
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        consumer.close();
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
            Thread.sleep(10);
        }
        // (1.a) start consumer again
        consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages1 = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages1.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client should not receive all produced messages and should be blocked due to unack-messages
        assertEquals(messages1.size(), receiverQueueSize);
        Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
            return (MessageIdImpl) m.getMessageId();
        }).collect(Collectors.toSet());
        // (3) redeliver all consumed messages
        consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
        Thread.sleep(1000);
        Set<MessageIdImpl> messages2 = Sets.newHashSet();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages2.add((MessageIdImpl) msg.getMessageId());
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        assertEquals(messages1.size(), messages2.size());
        // (4) Verify: redelivered all previous unacked-consumed messages
        messages2.removeAll(redeliveryMessages);
        assertEquals(messages2.size(), 0);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}

Example 12 with ProducerConfiguration

use of org.apache.pulsar.client.api.ProducerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testBackoffAndReconnect.

@Test(dataProvider = "batch")
public void testBackoffAndReconnect(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    // Create consumer and producer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic4", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic4", producerConf);
    // Produce messages
    CompletableFuture<MessageId> lastFuture = null;
    for (int i = 0; i < 10; i++) {
        lastFuture = producer.sendAsync(("my-message-" + i).getBytes()).thenApply(msgId -> {
            log.info("Published message id: {}", msgId);
            return msgId;
        });
    }
    lastFuture.get();
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
    }
    // Restart the broker and wait for the backoff to kick in. The client library will try to reconnect, and once
    // the broker is up, the consumer should receive the duplicate messages.
    log.info("-- Restarting broker --");
    restartBroker();
    msg = null;
    log.info("Receiving duplicate messages..");
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
        Assert.assertNotNull(msg, "Message cannot be null");
    }
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 13 with ProducerConfiguration

use of org.apache.pulsar.client.api.ProducerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testPriorityConsumer.

@Test
public void testPriorityConsumer() throws Exception {
    log.info("-- Starting {} test --", methodName);
    ConsumerConfiguration conf1 = new ConsumerConfiguration();
    conf1.setSubscriptionType(SubscriptionType.Shared);
    conf1.setPriorityLevel(1);
    conf1.setReceiverQueueSize(5);
    ConsumerConfiguration conf4 = new ConsumerConfiguration();
    conf4.setSubscriptionType(SubscriptionType.Shared);
    conf4.setPriorityLevel(2);
    conf4.setReceiverQueueSize(5);
    Consumer consumer1 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    Consumer consumer2 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    Consumer consumer3 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf1);
    Consumer consumer4 = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic2", "my-subscriber-name", conf4);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic2", producerConf);
    List<Future<MessageId>> futures = Lists.newArrayList();
    // Asynchronously produce messages
    for (int i = 0; i < 15; i++) {
        final String message = "my-message-" + i;
        Future<MessageId> future = producer.sendAsync(message.getBytes());
        futures.add(future);
    }
    log.info("Waiting for async publish to complete");
    for (Future<MessageId> future : futures) {
        future.get();
    }
    for (int i = 0; i < 20; i++) {
        consumer1.receive(100, TimeUnit.MILLISECONDS);
        consumer2.receive(100, TimeUnit.MILLISECONDS);
    }
    /**
     * a. consumer1 and consumer2 now has more permits (as received and sent more permits) b. try to produce more
     * messages: which will again distribute among consumer1 and consumer2 and should not dispatch to consumer4
     */
    for (int i = 0; i < 5; i++) {
        final String message = "my-message-" + i;
        Future<MessageId> future = producer.sendAsync(message.getBytes());
        futures.add(future);
    }
    Assert.assertNull(consumer4.receive(100, TimeUnit.MILLISECONDS));
    // Asynchronously acknowledge upto and including the last message
    producer.close();
    consumer1.close();
    consumer2.close();
    consumer3.close();
    consumer4.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 14 with ProducerConfiguration

use of org.apache.pulsar.client.api.ProducerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testEnabledChecksumClient.

@Test
public void testEnabledChecksumClient() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int totalMsg = 10;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic1", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    final int batchMessageDelayMs = 300;
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
    for (int i = 0; i < totalMsg; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Message msg = null;
    Set<String> messageSet = Sets.newHashSet();
    for (int i = 0; i < totalMsg; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
    }
    // Acknowledge the consumption of all messages at once
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}

Example 15 with ProducerConfiguration

use of org.apache.pulsar.client.api.ProducerConfiguration in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testConsumerBlockingWithUnAckedMessagesMultipleIteration.

/**
 * Verify: iteration of a. message receive w/o acking b. stop receiving msg c. ack msgs d. started receiving msgs
 *
 * 1. Produce total X (1500) messages 2. Consumer consumes messages without acking until stop receiving from broker
 * due to reaching ack-threshold (500) 3. Consumer acks messages after stop getting messages 4. Consumer again tries
 * to consume messages 5. Consumer should be able to complete consuming all 1500 messages in 3 iteration (1500/500)
 *
 * @throws Exception
 */
@Test
public void testConsumerBlockingWithUnAckedMessagesMultipleIteration() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 500;
        final int receiverQueueSize = 10;
        final int totalProducedMsgs = 1500;
        // receiver consumes messages in iteration after acknowledging broker
        final int totalReceiveIteration = totalProducedMsgs / unAckedMessagesBufferSize;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
        }
        int totalReceivedMessages = 0;
        // (2) Receive Messages
        for (int j = 0; j < totalReceiveIteration; j++) {
            Message msg = null;
            List<Message> messages = Lists.newArrayList();
            for (int i = 0; i < totalProducedMsgs; i++) {
                msg = consumer.receive(1, TimeUnit.SECONDS);
                if (msg != null) {
                    messages.add(msg);
                    log.info("Received message: " + new String(msg.getData()));
                } else {
                    break;
                }
            }
            // client must receive number of messages = unAckedMessagesBufferSize rather all produced messages
            assertEquals(messages.size(), unAckedMessagesBufferSize);
            // start acknowledging messages
            messages.forEach(m -> {
                try {
                    consumer.acknowledge(m);
                } catch (PulsarClientException e) {
                    fail("ack failed", e);
                }
            });
            totalReceivedMessages += messages.size();
        }
        // total received-messages should match to produced messages
        assertEquals(totalReceivedMessages, totalProducedMsgs);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}