Examples with MessageId org.apache.pulsar.client.api.MessageId used on opensource projects

Example 41 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class PulsarKafkaProducer method send.

@Override
public Future<RecordMetadata> send(ProducerRecord<K, V> record, Callback callback) {
    org.apache.pulsar.client.api.Producer<byte[]> producer;
    try {
        producer = producers.computeIfAbsent(record.topic(), topic -> createNewProducer(topic));
    } catch (Exception e) {
        if (callback != null) {
            callback.onCompletion(null, e);
        }
        CompletableFuture<RecordMetadata> future = new CompletableFuture<>();
        future.completeExceptionally(e);
        return future;
    }
    Message<byte[]> msg = getMessage(record);
    int messageSize = msg.getData().length;
    CompletableFuture<RecordMetadata> future = new CompletableFuture<>();
    CompletableFuture<MessageId> sendFuture = producer.sendAsync(msg);
    lastSendFuture.put(record.topic(), sendFuture);
    sendFuture.thenAccept((messageId) -> {
        future.complete(getRecordMetadata(record.topic(), msg, messageId, messageSize));
    }).exceptionally(ex -> {
        future.completeExceptionally(ex);
        return null;
    });
    future.handle((recordMetadata, throwable) -> {
        if (callback != null) {
            Exception exception = throwable != null ? new Exception(throwable) : null;
            callback.onCompletion(recordMetadata, exception);
        }
        return null;
    });
    return future;
}

Example 42 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class ConsumerImpl method redeliverUnacknowledgedMessages.

@Override
public void redeliverUnacknowledgedMessages(Set<MessageId> messageIds) {
    checkArgument(messageIds.stream().findFirst().get() instanceof MessageIdImpl);
    if (conf.getSubscriptionType() != SubscriptionType.Shared) {
        // We cannot redeliver single messages if subscription type is not Shared
        redeliverUnacknowledgedMessages();
        return;
    }
    ClientCnx cnx = cnx();
    if (isConnected() && cnx.getRemoteEndpointProtocolVersion() >= ProtocolVersion.v2.getNumber()) {
        int messagesFromQueue = removeExpiredMessagesFromQueue(messageIds);
        Iterable<List<MessageIdImpl>> batches = Iterables.partition(messageIds.stream().map(messageId -> (MessageIdImpl) messageId).collect(Collectors.toSet()), MAX_REDELIVER_UNACKNOWLEDGED);
        MessageIdData.Builder builder = MessageIdData.newBuilder();
        batches.forEach(ids -> {
            List<MessageIdData> messageIdDatas = ids.stream().map(messageId -> {
                // attempt to remove message from batchMessageAckTracker
                builder.setPartition(messageId.getPartitionIndex());
                builder.setLedgerId(messageId.getLedgerId());
                builder.setEntryId(messageId.getEntryId());
                return builder.build();
            }).collect(Collectors.toList());
            ByteBuf cmd = Commands.newRedeliverUnacknowledgedMessages(consumerId, messageIdDatas);
            cnx.ctx().writeAndFlush(cmd, cnx.ctx().voidPromise());
            messageIdDatas.forEach(MessageIdData::recycle);
        });
        if (messagesFromQueue > 0) {
            increaseAvailablePermits(cnx, messagesFromQueue);
        }
        builder.recycle();
        if (log.isDebugEnabled()) {
            log.debug("[{}] [{}] [{}] Redeliver unacked messages and increase {} permits", subscription, topic, consumerName, messagesFromQueue);
        }
        return;
    }
    if (cnx == null || (getState() == State.Connecting)) {
        log.warn("[{}] Client Connection needs to be establised for redelivery of unacknowledged messages", this);
    } else {
        log.warn("[{}] Reconnecting the client to redeliver the messages.", this);
        cnx.ctx().close();
    }
}

Example 43 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class ConsumerImpl method getLastMessageIdAsync.

CompletableFuture<MessageId> getLastMessageIdAsync() {
    if (getState() == State.Closing || getState() == State.Closed) {
        return FutureUtil.failedFuture(new PulsarClientException.AlreadyClosedException("Consumer was already closed"));
    }
    AtomicLong opTimeoutMs = new AtomicLong(client.getConfiguration().getOperationTimeoutMs());
    Backoff backoff = new Backoff(100, TimeUnit.MILLISECONDS, opTimeoutMs.get() * 2, TimeUnit.MILLISECONDS, 0, TimeUnit.MILLISECONDS);
    CompletableFuture<MessageId> getLastMessageIdFuture = new CompletableFuture<>();
    internalGetLastMessageIdAsync(backoff, opTimeoutMs, getLastMessageIdFuture);
    return getLastMessageIdFuture;
}

Example 44 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class ResendRequestTest method testExclusiveSingleAckedNormalTopic.

@Test(timeOut = testTimeout)
public void testExclusiveSingleAckedNormalTopic() throws Exception {
    String key = "testExclusiveSingleAckedNormalTopic";
    final String topicName = "persistent://prop/use/ns-abc/topic-" + key;
    final String subscriptionName = "my-ex-subscription-" + key;
    final String messagePredicate = "my-message-" + key + "-";
    final int totalMessages = 10;
    HashSet<MessageId> messageIdHashSet = new HashSet<MessageId>();
    HashSet<String> messageDataHashSet = new HashSet<String>();
    // 1. producer connect
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    // 2. Create consumer
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName(subscriptionName).receiverQueueSize(7).subscribe();
    // 3. producer publish messages
    for (int i = 0; i < totalMessages; i++) {
        String message = messagePredicate + i;
        producer.send(message.getBytes());
    }
    // 4. Receive messages
    Message<byte[]> message = consumer.receive();
    log.info("Message received " + new String(message.getData()));
    for (int i = 1; i < totalMessages; i++) {
        Message<byte[]> msg = consumer.receive();
        log.info("Message received " + new String(msg.getData()));
        messageDataHashSet.add(new String(msg.getData()));
    }
    printIncomingMessageQueue(consumer);
    // 5. Ack 1st message and ask for resend
    consumer.acknowledge(message);
    log.info("Message acked " + new String(message.getData()));
    messageIdHashSet.add(message.getMessageId());
    messageDataHashSet.add(new String(message.getData()));
    consumer.redeliverUnacknowledgedMessages();
    log.info("Resend Messages Request sent");
    // 6. Check if messages resent in correct order
    for (int i = 0; i < totalMessages - 1; i++) {
        message = consumer.receive();
        log.info("Message received " + new String(message.getData()));
        if (i < 2) {
            messageIdHashSet.add(message.getMessageId());
            consumer.acknowledge(message);
        }
        log.info("Message acked " + new String(message.getData()));
        assertTrue(messageDataHashSet.contains(new String(message.getData())));
    }
    assertEquals(messageIdHashSet.size(), 3);
    assertEquals(messageDataHashSet.size(), totalMessages);
    printIncomingMessageQueue(consumer);
    // 7. Request resend 2nd time - you should receive 4 messages
    consumer.redeliverUnacknowledgedMessages();
    log.info("Resend Messages Request sent");
    message = consumer.receive(2000, TimeUnit.MILLISECONDS);
    while (message != null) {
        log.info("Message received " + new String(message.getData()));
        consumer.acknowledge(message);
        log.info("Message acked " + new String(message.getData()));
        messageIdHashSet.add(message.getMessageId());
        messageDataHashSet.add(new String(message.getData()));
        message = consumer.receive(5000, TimeUnit.MILLISECONDS);
    }
    assertEquals(messageIdHashSet.size(), totalMessages);
    assertEquals(messageDataHashSet.size(), totalMessages);
    printIncomingMessageQueue(consumer);
    // 9. Calling resend after acking all messages - expectin 0 messages
    consumer.redeliverUnacknowledgedMessages();
    assertEquals(consumer.receive(2000, TimeUnit.MILLISECONDS), null);
    // 10. Checking message contents
    for (int i = 0; i < totalMessages; i++) {
        assertTrue(messageDataHashSet.contains(messagePredicate + i));
    }
}

Example 45 with MessageId

use of org.apache.pulsar.client.api.MessageId in project incubator-pulsar by apache.

the class SubscriptionSeekTest method testSeek.

@Test
public void testSeek() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/testSeek";
    Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
    // Disable pre-fetch in consumer to track the messages received
    org.apache.pulsar.client.api.Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-subscription").receiverQueueSize(0).subscribe();
    PersistentTopic topicRef = (PersistentTopic) pulsar.getBrokerService().getTopicReference(topicName);
    assertNotNull(topicRef);
    assertEquals(topicRef.getProducers().size(), 1);
    assertEquals(topicRef.getSubscriptions().size(), 1);
    List<MessageId> messageIds = new ArrayList<>();
    for (int i = 0; i < 10; i++) {
        String message = "my-message-" + i;
        MessageId msgId = producer.send(message.getBytes());
        messageIds.add(msgId);
    }
    PersistentSubscription sub = topicRef.getSubscription("my-subscription");
    assertEquals(sub.getNumberOfEntriesInBacklog(), 10);
    consumer.seek(MessageId.latest);
    assertEquals(sub.getNumberOfEntriesInBacklog(), 0);
    // Wait for consumer to reconnect
    Thread.sleep(500);
    consumer.seek(MessageId.earliest);
    assertEquals(sub.getNumberOfEntriesInBacklog(), 10);
    Thread.sleep(500);
    consumer.seek(messageIds.get(5));
    assertEquals(sub.getNumberOfEntriesInBacklog(), 5);
}