Examples with MessageId org.apache.pulsar.client.api.MessageId used on opensource projects

Example 16 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class PulsarSpout method fail.

@Override
public void fail(Object msgId) {
    if (msgId instanceof Message) {
        @SuppressWarnings("unchecked") Message<byte[]> msg = (Message<byte[]>) msgId;
        MessageId id = msg.getMessageId();
        LOG.warn("[{}] Error processing message {}", spoutId, id);
        // Since the message processing failed, we put it in the failed messages queue if there are more retries
        // remaining for the message
        MessageRetries messageRetries = pendingMessageRetries.computeIfAbsent(id, (k) -> new MessageRetries());
        if ((failedRetriesTimeoutNano < 0 || (messageRetries.getTimeStamp() + failedRetriesTimeoutNano) > System.nanoTime()) && (maxFailedRetries < 0 || messageRetries.numRetries < maxFailedRetries)) {
            // since we can retry again, we increment retry count and put it in the queue
            LOG.info("[{}] Putting message {} in the retry queue", spoutId, id);
            messageRetries.incrementAndGet();
            pendingMessageRetries.putIfAbsent(id, messageRetries);
            failedMessages.add(msg);
            --pendingAcks;
        } else {
            LOG.warn("[{}] Number of retries limit reached, dropping the message {}", spoutId, id);
            ack(msg);
        }
    }
}

Example 17 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class PersistentQueueE2ETest method testUnackedCountWithRedeliveries.

@Test
public void testUnackedCountWithRedeliveries() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/testUnackedCountWithRedeliveries";
    final String subName = "sub3";
    final int numMsgs = 10;
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    ConsumerBuilder<byte[]> consumerBuilder = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).receiverQueueSize(10).subscriptionType(SubscriptionType.Shared);
    ConsumerImpl<byte[]> consumer1 = (ConsumerImpl<byte[]>) consumerBuilder.subscribe();
    for (int i = 0; i < numMsgs; i++) {
        producer.send(("hello-" + i).getBytes());
    }
    Set<MessageId> c1_receivedMessages = new HashSet<>();
    // C-1 gets all messages but doesn't ack
    for (int i = 0; i < numMsgs; i++) {
        c1_receivedMessages.add(consumer1.receive().getMessageId());
    }
    // C-2 will not get any message initially, since everything went to C-1 already
    Consumer<byte[]> consumer2 = consumerBuilder.subscribe();
    // Trigger C-1 to redeliver everything, half will go C-1 again and the other half to C-2
    consumer1.redeliverUnacknowledgedMessages(c1_receivedMessages);
    // Consumer 2 will also receive all message but not ack
    for (int i = 0; i < numMsgs; i++) {
        consumer2.receive();
    }
    for (MessageId msgId : c1_receivedMessages) {
        consumer1.acknowledge(msgId);
    }
    PersistentTopicStats stats = admin.persistentTopics().getStats(topicName);
    // Unacked messages count should be 0 for both consumers at this point
    SubscriptionStats subStats = stats.subscriptions.get(subName);
    assertEquals(subStats.msgBacklog, 0);
    for (ConsumerStats cs : subStats.consumers) {
        assertEquals(cs.unackedMessages, 0);
    }
    producer.close();
    consumer1.close();
    consumer2.close();
    admin.persistentTopics().delete(topicName);
}

Example 18 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class PersistentTopicE2ETest method testProducerReturnedMessageId.

@Test
public void testProducerReturnedMessageId() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/topic-xyz";
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topicRef.getManagedLedger();
    long ledgerId = managedLedger.getLedgersInfoAsList().get(0).getLedgerId();
    // 2. producer publish messages
    final int SyncMessages = 10;
    for (int i = 0; i < SyncMessages; i++) {
        String message = "my-message-" + i;
        MessageId receivedMessageId = producer.send(message.getBytes());
        assertEquals(receivedMessageId, new MessageIdImpl(ledgerId, i, -1));
    }
    // 3. producer publish messages async
    final int AsyncMessages = 10;
    final CountDownLatch counter = new CountDownLatch(AsyncMessages);
    for (int i = SyncMessages; i < (SyncMessages + AsyncMessages); i++) {
        String content = "my-message-" + i;
        Message<byte[]> msg = MessageBuilder.create().setContent(content.getBytes()).build();
        final int index = i;
        producer.sendAsync(msg).thenRun(() -> {
            assertEquals(msg.getMessageId(), new MessageIdImpl(ledgerId, index, -1));
            counter.countDown();
        }).exceptionally((ex) -> {
            return null;
        });
    }
    counter.await();
    // 4. producer disconnect
    producer.close();
}

Example 19 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testSendCallBack.

@Test
public void testSendCallBack() throws Exception {
    log.info("-- Starting {} test --", methodName);
    final int totalMsg = 100;
    final ProducerConfiguration producerConf = new ProducerConfiguration();
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic1", producerConf);
    for (int i = 0; i < totalMsg; i++) {
        final String message = "my-message-" + i;
        Message msg = MessageBuilder.create().setContent(message.getBytes()).build();
        final AtomicInteger msgLength = new AtomicInteger();
        CompletableFuture<MessageId> future = producer.sendAsync(msg).handle((r, ex) -> {
            if (ex != null) {
                log.error("Message send failed:", ex);
            } else {
                msgLength.set(msg.getData().length);
            }
            return null;
        });
        future.get();
        assertEquals(message.getBytes().length, msgLength.get());
    }
}

Example 20 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testBackoffAndReconnect.

@Test(dataProvider = "batch")
public void testBackoffAndReconnect(int batchMessageDelayMs) throws Exception {
    log.info("-- Starting {} test --", methodName);
    // Create consumer and producer
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe("persistent://my-property/use/my-ns/my-topic4", "my-subscriber-name", conf);
    ProducerConfiguration producerConf = new ProducerConfiguration();
    if (batchMessageDelayMs != 0) {
        producerConf.setBatchingMaxPublishDelay(batchMessageDelayMs, TimeUnit.MILLISECONDS);
        producerConf.setBatchingMaxMessages(5);
        producerConf.setBatchingEnabled(true);
    }
    Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/my-topic4", producerConf);
    // Produce messages
    CompletableFuture<MessageId> lastFuture = null;
    for (int i = 0; i < 10; i++) {
        lastFuture = producer.sendAsync(("my-message-" + i).getBytes()).thenApply(msgId -> {
            log.info("Published message id: {}", msgId);
            return msgId;
        });
    }
    lastFuture.get();
    Message msg = null;
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
    }
    // Restart the broker and wait for the backoff to kick in. The client library will try to reconnect, and once
    // the broker is up, the consumer should receive the duplicate messages.
    log.info("-- Restarting broker --");
    restartBroker();
    msg = null;
    log.info("Receiving duplicate messages..");
    for (int i = 0; i < 10; i++) {
        msg = consumer.receive(5, TimeUnit.SECONDS);
        log.info("Received: [{}]", new String(msg.getData()));
        Assert.assertNotNull(msg, "Message cannot be null");
    }
    consumer.acknowledgeCumulative(msg);
    consumer.close();
    log.info("-- Exiting {} test --", methodName);
}