Examples with MessageMetadata org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata used on opensource projects

Example 16 with MessageMetadata

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata in project incubator-pulsar by apache.

the class ConsumerImpl method messageReceived.

void messageReceived(MessageIdData messageId, ByteBuf headersAndPayload, ClientCnx cnx) {
    if (log.isDebugEnabled()) {
        log.debug("[{}][{}] Received message: {}/{}", topic, subscription, messageId.getLedgerId(), messageId.getEntryId());
    }
    MessageMetadata msgMetadata = null;
    ByteBuf payload = headersAndPayload;
    if (!verifyChecksum(headersAndPayload, messageId)) {
        // discard message with checksum error
        discardCorruptedMessage(messageId, cnx, ValidationError.ChecksumMismatch);
        return;
    }
    try {
        msgMetadata = Commands.parseMessageMetadata(payload);
    } catch (Throwable t) {
        discardCorruptedMessage(messageId, cnx, ValidationError.ChecksumMismatch);
        return;
    }
    ByteBuf decryptedPayload = decryptPayloadIfNeeded(messageId, msgMetadata, payload, cnx);
    if (decryptedPayload == null) {
        // Message was discarded or CryptoKeyReader isn't implemented
        return;
    }
    ByteBuf uncompressedPayload = uncompressPayloadIfNeeded(messageId, msgMetadata, decryptedPayload, cnx);
    decryptedPayload.release();
    if (uncompressedPayload == null) {
        // Message was discarded on decompression error
        return;
    }
    final int numMessages = msgMetadata.getNumMessagesInBatch();
    if (numMessages == 1 && !msgMetadata.hasNumMessagesInBatch()) {
        final MessageImpl<T> message = new MessageImpl<>(messageId, msgMetadata, uncompressedPayload, getPartitionIndex(), cnx, schema);
        uncompressedPayload.release();
        msgMetadata.recycle();
        lock.readLock().lock();
        try {
            // Enqueue the message so that it can be retrieved when application calls receive()
            // if the conf.getReceiverQueueSize() is 0 then discard message if no one is waiting for it.
            // if asyncReceive is waiting then notify callback without adding to incomingMessages queue
            unAckedMessageTracker.add((MessageIdImpl) message.getMessageId());
            boolean asyncReceivedWaiting = !pendingReceives.isEmpty();
            if ((conf.getReceiverQueueSize() != 0 || waitingOnReceiveForZeroQueueSize) && !asyncReceivedWaiting) {
                incomingMessages.add(message);
            }
            if (asyncReceivedWaiting) {
                notifyPendingReceivedCallback(message, null);
            }
        } finally {
            lock.readLock().unlock();
        }
    } else {
        if (conf.getReceiverQueueSize() == 0) {
            log.warn("Closing consumer [{}]-[{}] due to unsupported received batch-message with zero receiver queue size", subscription, consumerName);
            // close connection
            closeAsync().handle((ok, e) -> {
                // notify callback with failure result
                notifyPendingReceivedCallback(null, new PulsarClientException.InvalidMessageException(format("Unsupported Batch message with 0 size receiver queue for [%s]-[%s] ", subscription, consumerName)));
                return null;
            });
        } else {
            // handle batch message enqueuing; uncompressed payload has all messages in batch
            receiveIndividualMessagesFromBatch(msgMetadata, uncompressedPayload, messageId, cnx);
        }
        uncompressedPayload.release();
        msgMetadata.recycle();
    }
    if (listener != null) {
        // Trigger the notification on the message listener in a separate thread to avoid blocking the networking
        // thread while the message processing happens
        listenerExecutor.execute(() -> {
            for (int i = 0; i < numMessages; i++) {
                try {
                    Message<T> msg = internalReceive(0, TimeUnit.MILLISECONDS);
                    // complete the callback-loop in case queue is cleared up
                    if (msg == null) {
                        if (log.isDebugEnabled()) {
                            log.debug("[{}] [{}] Message has been cleared from the queue", topic, subscription);
                        }
                        break;
                    }
                    try {
                        if (log.isDebugEnabled()) {
                            log.debug("[{}][{}] Calling message listener for message {}", topic, subscription, msg.getMessageId());
                        }
                        listener.received(ConsumerImpl.this, msg);
                    } catch (Throwable t) {
                        log.error("[{}][{}] Message listener error in processing message: {}", topic, subscription, msg.getMessageId(), t);
                    }
                } catch (PulsarClientException e) {
                    log.warn("[{}] [{}] Failed to dequeue the message for listener", topic, subscription, e);
                    return;
                }
            }
        });
    }
}

Example 17 with MessageMetadata

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata in project incubator-pulsar by apache.

the class ServerCnxTest method testSendSuccessOnEncryptionRequiredTopic.

@Test(timeOut = 30000)
public void testSendSuccessOnEncryptionRequiredTopic() throws Exception {
    resetChannel();
    setChannelConnected();
    // Set encryption_required to true
    ZooKeeperDataCache<Policies> zkDataCache = mock(ZooKeeperDataCache.class);
    Policies policies = mock(Policies.class);
    policies.encryption_required = true;
    policies.clusterDispatchRate = Maps.newHashMap();
    doReturn(Optional.of(policies)).when(zkDataCache).get(AdminResource.path(POLICIES, TopicName.get(encryptionRequiredTopicName).getNamespace()));
    doReturn(CompletableFuture.completedFuture(Optional.of(policies))).when(zkDataCache).getAsync(AdminResource.path(POLICIES, TopicName.get(encryptionRequiredTopicName).getNamespace()));
    doReturn(zkDataCache).when(configCacheService).policiesCache();
    ByteBuf clientCommand = Commands.newProducer(encryptionRequiredTopicName, 1, /* producer id */
    1, /* request id */
    "prod-name", true, null);
    channel.writeInbound(clientCommand);
    assertTrue(getResponse() instanceof CommandProducerSuccess);
    // test success case: encrypted messages can be published
    MessageMetadata messageMetadata = MessageMetadata.newBuilder().setPublishTime(System.currentTimeMillis()).setProducerName("prod-name").setSequenceId(0).addEncryptionKeys(EncryptionKeys.newBuilder().setKey("testKey").setValue(ByteString.copyFrom("testVal".getBytes()))).build();
    ByteBuf data = Unpooled.buffer(1024);
    clientCommand = ByteBufPair.coalesce(Commands.newSend(1, 0, 1, ChecksumType.None, messageMetadata, data));
    channel.writeInbound(Unpooled.copiedBuffer(clientCommand));
    clientCommand.release();
    assertTrue(getResponse() instanceof CommandSendReceipt);
    channel.finish();
}

Example 18 with MessageMetadata

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata in project incubator-pulsar by apache.

the class CompactedOutBatchMessageTest method testCompactedOutMessages.

@Test
public void testCompactedOutMessages() throws Exception {
    final String ns1 = "my-property/use/con-ns1";
    admin.namespaces().createNamespace(ns1);
    final String topic1 = "persistent://" + ns1 + "/my-topic";
    MessageMetadata metadata = MessageMetadata.newBuilder().setProducerName("foobar").setSequenceId(1).setPublishTime(1).setNumMessagesInBatch(3).build();
    // build a buffer with 4 messages, first and last compacted out
    ByteBuf batchBuffer = Unpooled.buffer(1000);
    Commands.serializeSingleMessageInBatchWithPayload(SingleMessageMetadata.newBuilder().setCompactedOut(true).setPartitionKey("key1"), Unpooled.EMPTY_BUFFER, batchBuffer);
    Commands.serializeSingleMessageInBatchWithPayload(SingleMessageMetadata.newBuilder().setCompactedOut(true).setPartitionKey("key2"), Unpooled.EMPTY_BUFFER, batchBuffer);
    Commands.serializeSingleMessageInBatchWithPayload(SingleMessageMetadata.newBuilder().setCompactedOut(false).setPartitionKey("key3"), Unpooled.EMPTY_BUFFER, batchBuffer);
    Commands.serializeSingleMessageInBatchWithPayload(SingleMessageMetadata.newBuilder().setCompactedOut(true).setPartitionKey("key4"), Unpooled.EMPTY_BUFFER, batchBuffer);
    try (ConsumerImpl<byte[]> consumer = (ConsumerImpl<byte[]>) pulsarClient.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe()) {
        // shove it in the sideways
        consumer.receiveIndividualMessagesFromBatch(metadata, batchBuffer, MessageIdData.newBuilder().setLedgerId(1234).setEntryId(567).build(), consumer.cnx());
        Message m = consumer.receive();
        assertEquals(((BatchMessageIdImpl) m.getMessageId()).getLedgerId(), 1234);
        assertEquals(((BatchMessageIdImpl) m.getMessageId()).getEntryId(), 567);
        assertEquals(((BatchMessageIdImpl) m.getMessageId()).getBatchIndex(), 2);
        assertEquals(m.getKey(), "key3");
        assertEquals(consumer.numMessagesInQueue(), 0);
    }
}

Example 19 with MessageMetadata

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata in project incubator-pulsar by apache.

the class ServerCnx method printSendCommandDebug.

private void printSendCommandDebug(CommandSend send, ByteBuf headersAndPayload) {
    headersAndPayload.markReaderIndex();
    MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload);
    headersAndPayload.resetReaderIndex();
    log.debug("[{}] Received send message request. producer: {}:{} {}:{} size: {}", remoteAddress, send.getProducerId(), send.getSequenceId(), msgMetadata.getProducerName(), msgMetadata.getSequenceId(), headersAndPayload.readableBytes());
    msgMetadata.recycle();
}

Example 20 with MessageMetadata

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata in project incubator-pulsar by apache.

the class MessageDeduplication method shouldPublishNextMessage.

/**
 * Assess whether the message was already stored in the topic.
 *
 * @return true if the message should be published or false if it was recognized as a duplicate
 */
public boolean shouldPublishNextMessage(PublishContext publishContext, ByteBuf headersAndPayload) {
    if (!isEnabled()) {
        return true;
    }
    String producerName = publishContext.getProducerName();
    long sequenceId = publishContext.getSequenceId();
    if (producerName.startsWith(replicatorPrefix)) {
        // Message is coming from replication, we need to use the original producer name and sequence id
        // for the purpose of deduplication and not rely on the "replicator" name.
        int readerIndex = headersAndPayload.readerIndex();
        MessageMetadata md = Commands.parseMessageMetadata(headersAndPayload);
        producerName = md.getProducerName();
        sequenceId = md.getSequenceId();
        publishContext.setOriginalProducerName(producerName);
        publishContext.setOriginalSequenceId(sequenceId);
        headersAndPayload.readerIndex(readerIndex);
        md.recycle();
    }
    // disconnects and re-connects very quickly. At that point the call can be coming from a different thread
    synchronized (highestSequencedPushed) {
        Long lastSequenceIdPushed = highestSequencedPushed.get(producerName);
        if (lastSequenceIdPushed != null && sequenceId <= lastSequenceIdPushed) {
            if (log.isDebugEnabled()) {
                log.debug("[{}] Message identified as duplicated producer={} seq-id={} -- highest-seq-id={}", topic.getName(), producerName, sequenceId, lastSequenceIdPushed);
            }
            return false;
        }
        highestSequencedPushed.put(producerName, sequenceId);
    }
    return true;
}