Examples with Message com.yahoo.pulsar.client.api.Message used on opensource projects

Example 51 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class ZeroQueueSizeTest method zeroQueueSizeNormalConsumer.

@Test()
public void zeroQueueSizeNormalConsumer() throws PulsarClientException {
    String key = "nonZeroQueueSizeNormalConsumer";
    // 1. Config
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    ConsumerConfiguration configuration = new ConsumerConfiguration();
    configuration.setReceiverQueueSize(0);
    ConsumerImpl consumer = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        log.info("Producer produced: " + message);
        producer.send(message.getBytes());
    }
    // 4. Receiver receives the message
    Message message;
    for (int i = 0; i < totalMessages; i++) {
        assertEquals(consumer.numMessagesInQueue(), 0);
        message = consumer.receive();
        assertEquals(new String(message.getData()), messagePredicate + i);
        assertEquals(consumer.numMessagesInQueue(), 0);
        log.info("Consumer received : " + new String(message.getData()));
    }
}

Example 52 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class ZeroQueueSizeTest method zeroQueueSizeSharedSubscription.

@Test()
public void zeroQueueSizeSharedSubscription() throws PulsarClientException {
    String key = "zeroQueueSizeSharedSubscription";
    // 1. Config
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    // 2. Create Producer
    Producer producer = pulsarClient.createProducer(topicName);
    // 3. Create Consumer
    int numOfSubscribers = 4;
    ConsumerConfiguration configuration = new ConsumerConfiguration();
    configuration.setReceiverQueueSize(0);
    configuration.setSubscriptionType(SubscriptionType.Shared);
    ConsumerImpl[] consumers = new ConsumerImpl[numOfSubscribers];
    for (int i = 0; i < numOfSubscribers; i++) {
        consumers[i] = (ConsumerImpl) pulsarClient.subscribe(topicName, subscriptionName, configuration);
    }
    // 4. Produce Messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
    }
    // 5. Consume messages
    Message message;
    for (int i = 0; i < totalMessages; i++) {
        assertEquals(consumers[i % numOfSubscribers].numMessagesInQueue(), 0);
        message = consumers[i % numOfSubscribers].receive();
        assertEquals(new String(message.getData()), messagePredicate + i);
        assertEquals(consumers[i % numOfSubscribers].numMessagesInQueue(), 0);
        log.info("Consumer received : " + new String(message.getData()));
    }
}

Example 53 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class PulsarSpout method fail.

@Override
public void fail(Object msgId) {
    if (msgId instanceof Message) {
        Message msg = (Message) msgId;
        MessageId id = msg.getMessageId();
        LOG.warn("[{}] Error processing message {}", spoutId, id);
        // Since the message processing failed, we put it in the failed messages queue if there are more retries
        // remaining for the message
        MessageRetries messageRetries = pendingMessageRetries.computeIfAbsent(id, (k) -> new MessageRetries());
        if ((failedRetriesTimeoutNano < 0 || (messageRetries.getTimeStamp() + failedRetriesTimeoutNano) > System.nanoTime()) && (maxFailedRetries < 0 || messageRetries.numRetries < maxFailedRetries)) {
            // since we can retry again, we increment retry count and put it in the queue
            LOG.info("[{}] Putting message {} in the retry queue", spoutId, id);
            messageRetries.incrementAndGet();
            pendingMessageRetries.putIfAbsent(id, messageRetries);
            failedMessages.add(msg);
            --pendingAcks;
        } else {
            LOG.warn("[{}] Number of retries limit reached, dropping the message {}", spoutId, id);
            ack(msg);
        }
    }
}

Example 54 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class PulsarSpout method ack.

@Override
public void ack(Object msgId) {
    if (msgId instanceof Message) {
        Message msg = (Message) msgId;
        if (LOG.isDebugEnabled()) {
            LOG.debug("[{}] Received ack for message {}", spoutId, msg.getMessageId());
        }
        consumer.acknowledgeAsync(msg);
        pendingMessageRetries.remove(msg.getMessageId());
        --pendingAcks;
    }
}

Example 55 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class PulsarSpout method nextTuple.

/**
     * Emits a tuple received from the Pulsar consumer unless there are any failed messages
     */
@Override
public void nextTuple() {
    Message msg;
    // check if there are any failed messages to re-emit in the topology
    msg = failedMessages.peek();
    if (msg != null) {
        MessageRetries messageRetries = pendingMessageRetries.get(msg.getMessageId());
        if (Backoff.shouldBackoff(messageRetries.getTimeStamp(), TimeUnit.NANOSECONDS, messageRetries.getNumRetries())) {
            Utils.sleep(100);
        } else {
            // remove the message from the queue and emit to the topology, only if it should not be backedoff
            LOG.info("[{}] Retrying failed message {}", spoutId, msg.getMessageId());
            failedMessages.remove();
            mapToValueAndEmit(msg);
        }
        return;
    }
    // receive from consumer if no failed messages
    if (consumer != null) {
        if (LOG.isDebugEnabled()) {
            LOG.debug("[{}] Receiving the next message from pulsar consumer to emit to the collector", spoutId);
        }
        try {
            msg = consumer.receive(1, TimeUnit.SECONDS);
            if (msg != null) {
                ++messagesReceived;
                messageSizeReceived += msg.getData().length;
            }
            mapToValueAndEmit(msg);
        } catch (PulsarClientException e) {
            LOG.error("[{}] Error receiving message from pulsar consumer", spoutId, e);
        }
    }
}