Examples with Producer com.yahoo.pulsar.client.api.Producer used on opensource projects

Example 96 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ResendRequestTest method testExclusiveCumulativeAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testExclusiveCumulativeAckedNormalTopic() throws Exception {
    String key = "testExclusiveCumulativeAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(7);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
    }
    // 4. Receive messages
    Message message = consumer.receive();
    log.info("Message received " + new String(message.getData()));
    for (int i = 0; i < 7; i++) {
        printIncomingMessageQueue(consumer);
        message = consumer.receive();
        log.info("Message received " + new String(message.getData()));
    }
    consumer.redeliverUnacknowledgedMessages();
    Thread.sleep(1000);
    consumer.acknowledgeCumulative(message);
    do {
        message = consumer.receive(1000, TimeUnit.MILLISECONDS);
    } while (message != null);
    log.info("Consumer Requests Messages");
    consumer.redeliverUnacknowledgedMessages();
    int numOfReceives = 0;
    message = consumer.receive();
    do {
        numOfReceives += 1;
        log.info("Message received " + new String(message.getData()));
        message = consumer.receive(1000, TimeUnit.MILLISECONDS);
    } while (message != null);
    assertEquals(numOfReceives, 2);
}

Example 97 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class MessageIdTest method producerSend.

@Test(timeOut = 10000)
public void producerSend() throws PulsarClientException {
    // 1. Basic Config
    String key = "producerSend";
    final String topicName = "persistent://prop/cluster/namespace/topic-" + key;
    final String subscriptionName = "my-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int numberOfMessages = 30;
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName);
    // 4. Publish message and get message id
    Set<MessageId> messageIds = new HashSet();
    for (int i = 0; i < numberOfMessages; i++) {
        String message = messagePredicate + i;
        messageIds.add(producer.send(message.getBytes()));
    }
    // 4. Check if message Ids are correct
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), numberOfMessages, "Not all messages published successfully");
    for (int i = 0; i < numberOfMessages; i++) {
        Assert.assertTrue(messageIds.remove(consumer.receive().getMessageId()), "Failed to receive Message");
    }
    log.info("Message IDs = " + messageIds);
    Assert.assertEquals(messageIds.size(), 0, "Not all messages received successfully");
    consumer.unsubscribe();
    ;
}

Example 98 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class UnAcknowledgedMessagesTimeoutTest method testFailoverSingleAckedPartitionedTopic.

@Test(timeOut = testTimeout)
public void testFailoverSingleAckedPartitionedTopic() throws Exception {
    String key = "testFailoverSingleAckedPartitionedTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-failover-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    final int numberOfPartitions = 3;
    admin.persistentTopics().createPartitionedTopic(topicName, numberOfPartitions);
    // Special step to create partitioned topic
    // 1. producer connect
    ProducerConfiguration prodConfig = new ProducerConfiguration();
    prodConfig.setMessageRoutingMode(MessageRoutingMode.RoundRobinPartition);
    Producer producer = pulsarClient.createProducer(topicName, prodConfig);
    // 2. Create consumer
    ConsumerConfiguration consumerConfig = new ConsumerConfiguration();
    consumerConfig.setReceiverQueueSize(7);
    consumerConfig.setSubscriptionType(SubscriptionType.Failover);
    consumerConfig.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    consumerConfig.setConsumerName("Consumer-1");
    Consumer consumer1 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    consumerConfig.setConsumerName("Consumer-2");
    Consumer consumer2 = pulsarClient.subscribe(topicName, subscriptionName, consumerConfig);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Message produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receive messages
    Message message1 = consumer1.receive();
    Message message2 = consumer2.receive();
    int messageCount1 = 0;
    int messageCount2 = 0;
    int ackCount1 = 0;
    int ackCount2 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
            consumer2.acknowledge(message2);
            ackCount2 += 1;
        }
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(messageCount1 + messageCount2, totalMessages);
    // 5. Check if Messages redelivered again
    // Since receive is a blocking call hoping that timeout will kick in
    log.info(key + " Timeout should be triggered now");
    message1 = consumer1.receive();
    messageCount1 = 0;
    do {
        if (message1 != null) {
            log.info("Consumer1 received " + new String(message1.getData()));
            messageCount1 += 1;
            consumer1.acknowledge(message1);
            ackCount1 += 1;
        }
        if (message2 != null) {
            log.info("Consumer2 received " + new String(message2.getData()));
            messageCount2 += 1;
        }
        message1 = consumer1.receive(500, TimeUnit.MILLISECONDS);
        message2 = consumer2.receive(500, TimeUnit.MILLISECONDS);
    } while (message1 != null || message2 != null);
    log.info(key + " messageCount1 = " + messageCount1);
    log.info(key + " messageCount2 = " + messageCount2);
    log.info(key + " ackCount1 = " + ackCount1);
    log.info(key + " ackCount2 = " + ackCount2);
    assertEquals(ackCount1 + messageCount2, totalMessages);
}

Example 99 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class UnAcknowledgedMessagesTimeoutTest method testExclusiveSingleAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testExclusiveSingleAckedNormalTopic() throws Exception {
    String key = "testExclusiveSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    // 1. producer connect
    Producer producer = pulsarClient.createProducer(topicName);
    // 2. Create consumer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setReceiverQueueSize(7);
    conf.setAckTimeout(ackTimeOutMillis, TimeUnit.MILLISECONDS);
    Consumer consumer = pulsarClient.subscribe(topicName, subscriptionName, conf);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages / 2; i++) {
        String message = messagePredicate + i;
        log.info("Producer produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message
    Message message = consumer.receive();
    while (message != null) {
        log.info("Consumer received : " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    }
    long size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, totalMessages / 2);
    // Blocking call, redeliver should kick in
    message = consumer.receive();
    log.info("Consumer received : " + new String(message.getData()));
    HashSet<String> hSet = new HashSet<>();
    for (int i = totalMessages / 2; i < totalMessages; i++) {
        String messageString = messagePredicate + i;
        producer.send(messageString.getBytes());
    }
    do {
        hSet.add(new String(message.getData()));
        consumer.acknowledge(message);
        log.info("Consumer acknowledged : " + new String(message.getData()));
        message = consumer.receive(500, TimeUnit.MILLISECONDS);
    } while (message != null);
    size = ((ConsumerImpl) consumer).getUnAckedMessageTracker().size();
    log.info(key + " Unacked Message Tracker size is " + size);
    assertEquals(size, 0);
    assertEquals(hSet.size(), totalMessages);
}

Example 100 with Producer

use of com.yahoo.pulsar.client.api.Producer in project pulsar by yahoo.

the class ProducerImpl method connectionOpened.

@Override
void connectionOpened(final ClientCnx cnx) {
    // we set the cnx reference before registering the producer on the cnx, so if the cnx breaks before creating the
    // producer, it will try to grab a new cnx
    setClientCnx(cnx);
    cnx.registerProducer(producerId, this);
    log.info("[{}] [{}] Creating producer on cnx {}", topic, producerName, cnx.ctx().channel());
    long requestId = client.newRequestId();
    cnx.sendRequestWithId(Commands.newProducer(topic, producerId, requestId, producerName), requestId).thenAccept(producerName -> {
        synchronized (ProducerImpl.this) {
            if (getState() == State.Closing || getState() == State.Closed) {
                cnx.removeProducer(producerId);
                cnx.channel().close();
                return;
            }
            resetBackoff();
            log.info("[{}] [{}] Created producer on cnx {}", topic, producerName, cnx.ctx().channel());
            connectionId = cnx.ctx().channel().toString();
            connectedSince = DATE_FORMAT.format(Instant.now());
            if (this.producerName == null) {
                this.producerName = producerName;
            }
            if (!producerCreatedFuture.isDone() && isBatchMessagingEnabled()) {
                client.timer().newTimeout(batchMessageAndSendTask, conf.getBatchingMaxPublishDelayMs(), TimeUnit.MILLISECONDS);
            }
            resendMessages(cnx);
        }
    }).exceptionally((e) -> {
        cnx.removeProducer(producerId);
        if (getState() == State.Closing || getState() == State.Closed) {
            // Producer was closed while reconnecting, close the connection to make sure the broker
            // drops the producer on its side
            cnx.channel().close();
            return null;
        }
        log.error("[{}] [{}] Failed to create producer: {}", topic, producerName, e.getCause().getMessage());
        if (e.getCause() instanceof PulsarClientException.ProducerBlockedQuotaExceededException) {
            synchronized (this) {
                log.warn("[{}] [{}] Topic backlog quota exceeded. Throwing Exception on producer.", topic, producerName);
                if (log.isDebugEnabled()) {
                    log.debug("[{}] [{}] Pending messages: {}", topic, producerName, pendingMessages.size());
                }
                PulsarClientException bqe = new PulsarClientException.ProducerBlockedQuotaExceededException("Could not send pending messages as backlog exceeded");
                failPendingMessages(cnx(), bqe);
            }
        } else if (e.getCause() instanceof PulsarClientException.ProducerBlockedQuotaExceededError) {
            log.warn("[{}] [{}] Producer is blocked on creation because backlog exceeded on topic.", producerName, topic);
        }
        if (//
        producerCreatedFuture.isDone() || (e.getCause() instanceof PulsarClientException && isRetriableError((PulsarClientException) e.getCause()) && System.currentTimeMillis() < createProducerTimeout)) {
            // Either we had already created the producer once (producerCreatedFuture.isDone()) or we are
            // still within the initial timeout budget and we are dealing with a retriable error
            reconnectLater(e.getCause());
        } else {
            setState(State.Failed);
            producerCreatedFuture.completeExceptionally(e.getCause());
            client.cleanupProducer(this);
        }
        return null;
    });
}