Examples with Message org.apache.pulsar.client.api.Message used on opensource projects

Example 11 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class PulsarSpout method fail.

@Override
public void fail(Object msgId) {
    if (msgId instanceof Message) {
        @SuppressWarnings("unchecked") Message<byte[]> msg = (Message<byte[]>) msgId;
        MessageId id = msg.getMessageId();
        LOG.warn("[{}] Error processing message {}", spoutId, id);
        // Since the message processing failed, we put it in the failed messages queue if there are more retries
        // remaining for the message
        MessageRetries messageRetries = pendingMessageRetries.computeIfAbsent(id, (k) -> new MessageRetries());
        if ((failedRetriesTimeoutNano < 0 || (messageRetries.getTimeStamp() + failedRetriesTimeoutNano) > System.nanoTime()) && (maxFailedRetries < 0 || messageRetries.numRetries < maxFailedRetries)) {
            // since we can retry again, we increment retry count and put it in the queue
            LOG.info("[{}] Putting message {} in the retry queue", spoutId, id);
            messageRetries.incrementAndGet();
            pendingMessageRetries.putIfAbsent(id, messageRetries);
            failedMessages.add(msg);
            --pendingAcks;
        } else {
            LOG.warn("[{}] Number of retries limit reached, dropping the message {}", spoutId, id);
            ack(msg);
        }
    }
}

Example 12 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class PulsarBoltTest method testNoMessageSend.

@Test
public void testNoMessageSend() throws Exception {
    String msgContent = "message to be dropped";
    Tuple tuple = getMockTuple(msgContent);
    bolt.execute(tuple);
    Assert.assertTrue(mockCollector.acked());
    Message msg = consumer.receive(5, TimeUnit.SECONDS);
    Assert.assertNull(msg);
}

Example 13 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class ResendRequestTest method printIncomingMessageQueue.

@SuppressWarnings("unchecked")
private BlockingQueue<Message<byte[]>> printIncomingMessageQueue(Consumer<byte[]> consumer) throws Exception {
    BlockingQueue<Message<byte[]>> imq = null;
    ConsumerBase<byte[]> c = (ConsumerBase<byte[]>) consumer;
    Field field = ConsumerBase.class.getDeclaredField("incomingMessages");
    field.setAccessible(true);
    imq = (BlockingQueue<Message<byte[]>>) field.get(c);
    log.info("Incoming MEssage Queue: {}", imq);
    return imq;
}

Example 14 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class PersistentTopicE2ETest method testConcurrentConsumerThreads.

// some race conditions needs to be handled
// disabling the test for now to not block commit jobs
@Test(enabled = false)
public void testConcurrentConsumerThreads() throws Exception {
    // test concurrent consumer threads on same consumerId
    final String topicName = "persistent://prop/use/ns-abc/topic3";
    final String subName = "sub3";
    final int recvQueueSize = 100;
    final int numConsumersThreads = 10;
    ExecutorService executor = Executors.newCachedThreadPool();
    final CyclicBarrier barrier = new CyclicBarrier(numConsumersThreads + 1);
    for (int i = 0; i < numConsumersThreads; i++) {
        executor.submit(new Callable<Void>() {

            @Override
            public Void call() throws Exception {
                barrier.await();
                Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subName).receiverQueueSize(recvQueueSize).subscribe();
                for (int i = 0; i < recvQueueSize / numConsumersThreads; i++) {
                    Message<byte[]> msg = consumer.receive();
                    consumer.acknowledge(msg);
                }
                return null;
            }
        });
    }
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    for (int i = 0; i < recvQueueSize * numConsumersThreads; i++) {
        String message = "my-message-" + i;
        producer.send(message.getBytes());
    }
    barrier.await();
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    PersistentSubscription subRef = topicRef.getSubscription(subName);
    // 1. cumulatively all threads drain the backlog
    assertEquals(subRef.getNumberOfEntriesInBacklog(), 0);
    // 2. flow control works the same as single consumer single thread
    Thread.sleep(ASYNC_EVENT_COMPLETION_WAIT);
    assertEquals(getAvailablePermits(subRef), recvQueueSize);
    executor.shutdown();
}

Example 15 with Message

use of org.apache.pulsar.client.api.Message in project incubator-pulsar by apache.

the class V1_ProducerConsumerTest method testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause.

/**
 * It verifies that redelivery-of-specific messages: that redelivers all those messages even when consumer gets
 * blocked due to unacked messsages
 *
 * Usecase: produce message with 10ms interval: so, consumer can consume only 10 messages without acking
 *
 * @throws Exception
 */
@Test
public void testBlockUnackedConsumerRedeliverySpecificMessagesProduceWithPause() throws Exception {
    log.info("-- Starting {} test --", methodName);
    int unAckedMessages = pulsar.getConfiguration().getMaxUnackedMessagesPerConsumer();
    try {
        final int unAckedMessagesBufferSize = 10;
        final int receiverQueueSize = 20;
        final int totalProducedMsgs = 20;
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessagesBufferSize);
        ConsumerConfiguration conf = new ConsumerConfiguration();
        conf.setReceiverQueueSize(receiverQueueSize);
        conf.setSubscriptionType(SubscriptionType.Shared);
        ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe("persistent://my-property/use/my-ns/unacked-topic", "subscriber-1", conf);
        ProducerConfiguration producerConf = new ProducerConfiguration();
        Producer producer = pulsarClient.createProducer("persistent://my-property/use/my-ns/unacked-topic", producerConf);
        // (1) Produced Messages
        for (int i = 0; i < totalProducedMsgs; i++) {
            String message = "my-message-" + i;
            producer.send(message.getBytes());
            Thread.sleep(10);
        }
        // (2) try to consume messages: but will be able to consume number of messages = unAckedMessagesBufferSize
        Message msg = null;
        List<Message> messages1 = Lists.newArrayList();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages1.add(msg);
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        // client should not receive all produced messages and should be blocked due to unack-messages
        assertEquals(messages1.size(), unAckedMessagesBufferSize);
        Set<MessageIdImpl> redeliveryMessages = messages1.stream().map(m -> {
            return (MessageIdImpl) m.getMessageId();
        }).collect(Collectors.toSet());
        // (3) redeliver all consumed messages
        consumer.redeliverUnacknowledgedMessages(Sets.newHashSet(redeliveryMessages));
        Thread.sleep(1000);
        Set<MessageIdImpl> messages2 = Sets.newHashSet();
        for (int i = 0; i < totalProducedMsgs; i++) {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                messages2.add((MessageIdImpl) msg.getMessageId());
                log.info("Received message: " + new String(msg.getData()));
            } else {
                break;
            }
        }
        assertEquals(messages1.size(), messages2.size());
        // (4) Verify: redelivered all previous unacked-consumed messages
        messages2.removeAll(redeliveryMessages);
        assertEquals(messages2.size(), 0);
        producer.close();
        consumer.close();
        log.info("-- Exiting {} test --", methodName);
    } catch (Exception e) {
        fail();
    } finally {
        pulsar.getConfiguration().setMaxUnackedMessagesPerConsumer(unAckedMessages);
    }
}