Examples with Producer com.yahoo.pulsar.client.api.Producer used on opensource projects

Example 51 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ZeroQueueSizeTest method zeroQueueSizeFailoverSubscription.

@Test()
public void zeroQueueSizeFailoverSubscription() throws PulsarClientException {
    String key = "zeroQueueSizeFailoverSubscription";
    // 1. Config
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    ConsumerConfiguration configuration = new ConsumerConfiguration();
    configuration.setReceiverQueueSize(0);
    configuration.setSubscriptionType(SubscriptionType.Failover);
    configuration.setConsumerName("consumer-1");
    ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
    configuration.setConsumerName("consumer-2");
    ConsumerImpl consumer2 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
    // 4. Produce Messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
    }
    // 5. Consume messages
    Message message;
    for (int i = 0; i < totalMessages / 2; i++) {
        assertEquals(consumer1.numMessagesInQueue(), 0);
        message = consumer1.receive();
        assertEquals(new String(message.getData()), messagePredicate + i);
        assertEquals(consumer1.numMessagesInQueue(), 0);
        log.info("Consumer received : " + new String(message.getData()));
    }
    // 6. Trigger redelivery
    consumer1.redeliverUnacknowledgedMessages();
    // 7. Trigger Failover
    consumer1.close();
    // 8. Receive messages on failed over consumer
    for (int i = 0; i < totalMessages / 2; i++) {
        assertEquals(consumer2.numMessagesInQueue(), 0);
        message = consumer2.receive();
        assertEquals(new String(message.getData()), messagePredicate + i);
        assertEquals(consumer2.numMessagesInQueue(), 0);
        log.info("Consumer received : " + new String(message.getData()));
    }
}

Example 52 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ZeroQueueSizeTest method testFailedZeroQueueSizeBatchMessage.

@Test()
public void testFailedZeroQueueSizeBatchMessage() throws PulsarClientException {
    int batchMessageDelayMs = 100;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Shared);
    conf.setReceiverQueueSize(0);
    Consumer consumer = pulsarClient.subscribe("persistent://prop-xyz/use/ns-abc/topic1", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
    }
    Producer producer = pulsarClient.createProducer("persistent://prop-xyz/use/ns-abc/topic1", producerConf);
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    try {
        consumer.receiveAsync().handle((ok, e) -> {
            if (e == null) {
                // as zero receiverQueueSize doesn't support batch message, must receive exception at callback.
                Assert.fail();
            }
            return null;
        });
    } finally {
        consumer.close();
    }
}

Example 53 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PartitionedProducerImpl method start.

private void start() {
    AtomicReference<Throwable> createFail = new AtomicReference<Throwable>();
    AtomicInteger completed = new AtomicInteger();
    for (int partitionIndex = 0; partitionIndex < numPartitions; partitionIndex++) {
        String partitionName = DestinationName.get(topic).getPartition(partitionIndex).toString();
        ProducerImpl producer = new ProducerImpl(client, partitionName, null, conf, new CompletableFuture<Producer>(), partitionIndex);
        producers.add(producer);
        producer.producerCreatedFuture().handle((prod, createException) -> {
            if (createException != null) {
                setState(State.Failed);
                createFail.compareAndSet(null, createException);
            }
            // created partitions
            if (completed.incrementAndGet() == numPartitions) {
                if (createFail.get() == null) {
                    setState(State.Ready);
                    producerCreatedFuture().complete(PartitionedProducerImpl.this);
                    log.info("[{}] Created partitioned producer", topic);
                } else {
                    closeAsync().handle((ok, closeException) -> {
                        producerCreatedFuture().completeExceptionally(createFail.get());
                        client.cleanupProducer(this);
                        return null;
                    });
                    log.error("[{}] Could not create partitioned producer.", topic, createFail.get().getCause());
                }
            }
            return null;
        });
    }
}

Example 54 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class PartitionedProducerImpl method closeAsync.

@Override
public CompletableFuture<Void> closeAsync() {
    if (getState() == State.Closing || getState() == State.Closed) {
        return CompletableFuture.completedFuture(null);
    }
    setState(State.Closing);
    AtomicReference<Throwable> closeFail = new AtomicReference<Throwable>();
    AtomicInteger completed = new AtomicInteger(numPartitions);
    CompletableFuture<Void> closeFuture = new CompletableFuture<>();
    for (Producer producer : producers) {
        if (producer != null) {
            producer.closeAsync().handle((closed, ex) -> {
                if (ex != null) {
                    closeFail.compareAndSet(null, ex);
                }
                if (completed.decrementAndGet() == 0) {
                    if (closeFail.get() == null) {
                        setState(State.Closed);
                        closeFuture.complete(null);
                        log.info("[{}] Closed Partitioned Producer", topic);
                        client.cleanupProducer(this);
                    } else {
                        setState(State.Failed);
                        closeFuture.completeExceptionally(closeFail.get());
                        log.error("[{}] Could not close Partitioned Producer", topic, closeFail.get().getCause());
                    }
                }
                return null;
            });
        }
    }
    return closeFuture;
}

Example 55 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class BrokerClientIntegrationTest method testUnsupportedBatchMessageConsumer.

/**
     * It verifies that consumer which doesn't support batch-message:
     * <p>
     * 1. broker disconnects that consumer
     * <p>
     * 2. redeliver all those messages to other supported consumer under the same subscription
     * 
     * @param subType
     * @throws Exception
     */
@Test(timeOut = 7000, dataProvider = "subType")
public void testUnsupportedBatchMessageConsumer(SubscriptionType subType) throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int batchMessageDelayMs = 1000;
    final String topicName = "persistent://my-property/use/my-ns/my-topic1";
    final String subscriptionName = "my-subscriber-name" + subType;
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(subType);
    ConsumerImpl consumer1 = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingEnabled(true);
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(20);
    }
    Producer producer = pulsarClient.createProducer(topicName, new ProducerConfiguration());
    Producer batchProducer = pulsarClient.createProducer(topicName, producerConf);
    // update consumer's version to incompatible batch-message version = Version.V3
    Topic topic = pulsar.getBrokerService().getTopic(topicName).get();
    com.yahoo.pulsar.broker.service.Consumer brokerConsumer = topic.getSubscriptions().get(subscriptionName).getConsumers().get(0);
    Field cnxField = com.yahoo.pulsar.broker.service.Consumer.class.getDeclaredField("cnx");
    cnxField.setAccessible(true);
    PulsarHandler cnx = (PulsarHandler) cnxField.get(brokerConsumer);
    Field versionField = PulsarHandler.class.getDeclaredField("remoteEndpointProtocolVersion");
    versionField.setAccessible(true);
    versionField.set(cnx, 3);
    // (1) send non-batch message: consumer should be able to consume
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    Set<String> messageSet = Sets.newHashSet();
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer1.receive(1, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        consumer1.acknowledge(msg);
    }
    // Also set clientCnx of the consumer to null so, it avoid reconnection so, other consumer can consume for
    // verification
    consumer1.setClientCnx(null);
    // (2) send batch-message which should not be able to consume: as broker will disconnect the consumer
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        batchProducer.sendAsync(message.getBytes());
    }
    Thread.sleep(batchMessageDelayMs);
    // consumer should have not received any message as it should have been disconnected
    msg = consumer1.receive(2, TimeUnit.SECONDS);
    assertNull(msg);
    // subscrie consumer2 with supporting batch version
    pulsarClient = PulsarClient.create(brokerUrl.toString());
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, conf);
    messageSet.clear();
    for (int i = 0; i < 10; i++) {
        msg = consumer2.receive(1, TimeUnit.SECONDS);
        String receivedMessage = new String(msg.getData());
        log.debug("Received message: [{}]", receivedMessage);
        String expectedMessage = "my-message-" + i;
        testMessageOrderAndDuplicates(messageSet, receivedMessage, expectedMessage);
        consumer2.acknowledge(msg);
    }
    consumer2.close();
    producer.close();
    batchProducer.close();
    log.info("-- Exiting {} test --", methodName);
}