Examples with Message com.yahoo.pulsar.client.api.Message used on opensource projects

Example 6 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class PartitionedConsumerImpl method internalReceiveAsync.

@Override
protected CompletableFuture<Message> internalReceiveAsync() {
    CompletableFuture<Message> result = new CompletableFuture<Message>();
    Message message;
    try {
        lock.writeLock().lock();
        message = incomingMessages.poll(0, TimeUnit.SECONDS);
        if (message == null) {
            pendingReceives.add(result);
        } else {
            resumeReceivingFromPausedConsumersIfNeeded();
            result.complete(message);
        }
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        result.completeExceptionally(new PulsarClientException(e));
    } finally {
        lock.writeLock().unlock();
    }
    return result;
}

Example 7 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class PartitionedConsumerImpl method messageReceived.

void messageReceived(Message message) {
    lock.readLock().lock();
    try {
        if (log.isDebugEnabled()) {
            log.debug("[{}][{}] Received message from partitioned-consumer {}", topic, subscription, message.getMessageId());
        }
        // if asyncReceive is waiting : return message to callback without adding to incomingMessages queue
        if (!pendingReceives.isEmpty()) {
            CompletableFuture<Message> receivedFuture = pendingReceives.poll();
            listenerExecutor.execute(() -> receivedFuture.complete(message));
        } else {
            // Enqueue the message so that it can be retrieved when application calls receive()
            // Waits for the queue to have space for the message
            incomingMessages.put(message);
        }
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
    } finally {
        lock.readLock().unlock();
    }
    if (listener != null) {
        // Trigger the notification on the message listener in a separate thread to avoid blocking the networking
        // thread while the message processing happens
        listenerExecutor.execute(() -> {
            Message msg;
            try {
                msg = internalReceive();
            } catch (PulsarClientException e) {
                log.warn("[{}] [{}] Failed to dequeue the message for listener", topic, subscription, e);
                return;
            }
            try {
                if (log.isDebugEnabled()) {
                    log.debug("[{}][{}] Calling message listener for message {}", topic, subscription, message.getMessageId());
                }
                listener.received(PartitionedConsumerImpl.this, msg);
            } catch (Throwable t) {
                log.error("[{}][{}] Message listener error in processing message: {}", topic, subscription, message, t);
            }
        });
    }
}

Example 8 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class AdminApiTest method persistentTopicsCursorResetAfterReset.

@Test(dataProvider = "topicName")
public void persistentTopicsCursorResetAfterReset(String topicName) throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    topicName = "persistent://prop-xyz/use/ns1/" + topicName;
    // create consumer and subscription
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, 5, 0);
    // Allow at least 1ms for messages to have different timestamps
    Thread.sleep(1);
    long firstTimestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 3, 5);
    Thread.sleep(1);
    long secondTimestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 2, 8);
    List<Message> messages = admin.persistentTopics().peekMessages(topicName, "my-sub", 10);
    assertEquals(messages.size(), 10);
    messages.forEach(message -> {
        LOG.info("Peeked message: {}", new String(message.getData()));
    });
    for (int i = 0; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
    }
    admin.persistentTopics().resetCursor(topicName, "my-sub", firstTimestamp);
    int receivedAfterReset = 0;
    // Should received messages from 4-9
    for (int i = 4; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
        ++receivedAfterReset;
        String expected = "message-" + i;
        assertEquals(new String(message.getData()), expected);
    }
    assertEquals(receivedAfterReset, 6);
    // Reset at 2nd timestamp
    receivedAfterReset = 0;
    admin.persistentTopics().resetCursor(topicName, "my-sub", secondTimestamp);
    // Should received messages from 7-9
    for (int i = 7; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
        ++receivedAfterReset;
        String expected = "message-" + i;
        assertEquals(new String(message.getData()), expected);
    }
    assertEquals(receivedAfterReset, 3);
    consumer.close();
    admin.persistentTopics().deleteSubscription(topicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList());
    admin.persistentTopics().delete(topicName);
}

Example 9 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class AdminApiTest method partitionedTopicsCursorReset.

@Test(dataProvider = "topicName")
public void partitionedTopicsCursorReset(String topicName) throws Exception {
    admin.namespaces().setRetention("prop-xyz/use/ns1", new RetentionPolicies(10, 10));
    topicName = "persistent://prop-xyz/use/ns1/" + topicName;
    admin.persistentTopics().createPartitionedTopic(topicName, 4);
    // create consumer and subscription
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = pulsarClient.subscribe(topicName, "my-sub", conf);
    List<String> destinations = admin.persistentTopics().getList("prop-xyz/use/ns1");
    assertEquals(destinations.size(), 4);
    assertEquals(admin.persistentTopics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic(topicName, 5, 0);
    Thread.sleep(1);
    long timestamp = System.currentTimeMillis();
    publishMessagesOnPersistentTopic(topicName, 5, 5);
    for (int i = 0; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
    }
    // messages should still be available due to retention
    admin.persistentTopics().resetCursor(topicName, "my-sub", timestamp);
    Set<String> expectedMessages = Sets.newHashSet();
    Set<String> receivedMessages = Sets.newHashSet();
    for (int i = 4; i < 10; i++) {
        Message message = consumer.receive();
        consumer.acknowledge(message);
        expectedMessages.add("message-" + i);
        receivedMessages.add(new String(message.getData()));
    }
    receivedMessages.removeAll(expectedMessages);
    assertEquals(receivedMessages.size(), 0);
    consumer.close();
    admin.persistentTopics().deleteSubscription(topicName, "my-sub");
    admin.persistentTopics().deletePartitionedTopic(topicName);
}

Example 10 with Message

use of com.yahoo.pulsar.client.api.Message in project pulsar by yahoo.

the class AdminApiTest method persistentTopics.

@Test(dataProvider = "topicName")
public void persistentTopics(String topicName) throws Exception {
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
    final String persistentTopicName = "persistent://prop-xyz/use/ns1/" + topicName;
    // Force to create a destination
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 0);
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList("persistent://prop-xyz/use/ns1/" + topicName));
    // create consumer and subscription
    URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
    ClientConfiguration clientConf = new ClientConfiguration();
    clientConf.setStatsInterval(0, TimeUnit.SECONDS);
    PulsarClient client = PulsarClient.create(pulsarUrl.toString(), clientConf);
    ConsumerConfiguration conf = new ConsumerConfiguration();
    conf.setSubscriptionType(SubscriptionType.Exclusive);
    Consumer consumer = client.subscribe(persistentTopicName, "my-sub", conf);
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList("my-sub"));
    publishMessagesOnPersistentTopic("persistent://prop-xyz/use/ns1/" + topicName, 10);
    PersistentTopicStats topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
    assertEquals(topicStats.publishers.size(), 0);
    PersistentTopicInternalStats internalStats = admin.persistentTopics().getInternalStats(persistentTopicName);
    assertEquals(internalStats.cursors.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
    List<Message> messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 3);
    assertEquals(messages.size(), 3);
    for (int i = 0; i < 3; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    messages = admin.persistentTopics().peekMessages(persistentTopicName, "my-sub", 15);
    assertEquals(messages.size(), 10);
    for (int i = 0; i < 10; i++) {
        String expectedMessage = "message-" + i;
        assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
    }
    admin.persistentTopics().skipMessages(persistentTopicName, "my-sub", 5);
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 5);
    admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
    consumer.close();
    client.close();
    admin.persistentTopics().deleteSubscription(persistentTopicName, "my-sub");
    assertEquals(admin.persistentTopics().getSubscriptions(persistentTopicName), Lists.newArrayList());
    topicStats = admin.persistentTopics().getStats(persistentTopicName);
    assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet());
    assertEquals(topicStats.publishers.size(), 0);
    try {
        admin.persistentTopics().skipAllMessages(persistentTopicName, "my-sub");
    } catch (NotFoundException e) {
    }
    admin.persistentTopics().delete(persistentTopicName);
    try {
        admin.persistentTopics().delete(persistentTopicName);
        fail("Should have received 404");
    } catch (NotFoundException e) {
    }
    assertEquals(admin.persistentTopics().getList("prop-xyz/use/ns1"), Lists.newArrayList());
}