Examples with MessageMetadata com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata used on opensource projects

Example 1 with MessageMetadata

use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata in project pulsar by yahoo.

the class ServerCnx method printSendCommandDebug.

private void printSendCommandDebug(CommandSend send, ByteBuf headersAndPayload) {
    headersAndPayload.markReaderIndex();
    MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload);
    headersAndPayload.resetReaderIndex();
    log.debug("[{}] Received send message request. producer: {}:{} {}:{} size: {}", remoteAddress, send.getProducerId(), send.getSequenceId(), msgMetadata.getProducerName(), msgMetadata.getSequenceId(), headersAndPayload.readableBytes());
    msgMetadata.recycle();
}

Example 2 with MessageMetadata

use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata in project pulsar by yahoo.

the class ServerCnxTest method testSendCommand.

@Test(timeOut = 30000)
public void testSendCommand() throws Exception {
    resetChannel();
    setChannelConnected();
    ByteBuf clientCommand = Commands.newProducer(successTopicName, 1, /* producer id */
    1, /* request id */
    "prod-name");
    channel.writeInbound(clientCommand);
    assertTrue(getResponse() instanceof CommandProducerSuccess);
    // test SEND success
    MessageMetadata messageMetadata = MessageMetadata.newBuilder().setPublishTime(System.currentTimeMillis()).setProducerName("prod-name").setSequenceId(0).build();
    ByteBuf data = Unpooled.buffer(1024);
    clientCommand = Commands.newSend(1, 0, 1, ChecksumType.None, messageMetadata, data);
    channel.writeInbound(Unpooled.copiedBuffer(clientCommand));
    clientCommand.release();
    assertTrue(getResponse() instanceof CommandSendReceipt);
    channel.finish();
}

Example 3 with MessageMetadata

use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata in project pulsar by yahoo.

the class ManagedLedgerTest method getMessageWithMetadata.

public ByteBuf getMessageWithMetadata(byte[] data) throws IOException {
    MessageMetadata messageData = MessageMetadata.newBuilder().setPublishTime(System.currentTimeMillis()).setProducerName("prod-name").setSequenceId(0).build();
    ByteBuf payload = Unpooled.wrappedBuffer(data, 0, data.length);
    int msgMetadataSize = messageData.getSerializedSize();
    int headersSize = 4 + msgMetadataSize;
    ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize, headersSize);
    ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
    headers.writeInt(msgMetadataSize);
    messageData.writeTo(outStream);
    outStream.recycle();
    return DoubleByteBuf.get(headers, payload);
}

Example 4 with MessageMetadata

use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata in project pulsar by yahoo.

the class PersistentTopics method peekNthMessage.

@GET
@Path("/{property}/{cluster}/{namespace}/{destination}/subscription/{subName}/position/{messagePosition}")
@ApiOperation(value = "Peek nth message on a topic subscription.")
@ApiResponses(value = { @ApiResponse(code = 403, message = "Don't have admin permission"), @ApiResponse(code = 404, message = "Topic, subscription or the message position does not exist") })
public Response peekNthMessage(@PathParam("property") String property, @PathParam("cluster") String cluster, @PathParam("namespace") String namespace, @PathParam("destination") @Encoded String destination, @PathParam("subName") String subName, @PathParam("messagePosition") int messagePosition, @QueryParam("authoritative") @DefaultValue("false") boolean authoritative) {
    destination = decode(destination);
    DestinationName dn = DestinationName.get(domain(), property, cluster, namespace, destination);
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(property, cluster, namespace, destination, authoritative);
    if (partitionMetadata.partitions > 0) {
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Peek messages on a partitioned topic is not allowed");
    }
    validateAdminOperationOnDestination(dn, authoritative);
    PersistentTopic topic = getTopicReference(dn);
    PersistentReplicator repl = null;
    PersistentSubscription sub = null;
    Entry entry = null;
    if (subName.startsWith(topic.replicatorPrefix)) {
        repl = getReplicatorReference(subName, topic);
    } else {
        sub = getSubscriptionReference(subName, topic);
    }
    try {
        if (subName.startsWith(topic.replicatorPrefix)) {
            entry = repl.peekNthMessage(messagePosition).get();
        } else {
            entry = sub.peekNthMessage(messagePosition).get();
        }
        checkNotNull(entry);
        PositionImpl pos = (PositionImpl) entry.getPosition();
        ByteBuf metadataAndPayload = entry.getDataBuffer();
        // moves the readerIndex to the payload
        MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
        ResponseBuilder responseBuilder = Response.ok();
        responseBuilder.header("X-Pulsar-Message-ID", pos.toString());
        for (KeyValue keyValue : metadata.getPropertiesList()) {
            responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue());
        }
        if (metadata.hasPublishTime()) {
            responseBuilder.header("X-Pulsar-publish-time", DATE_FORMAT.format(Instant.ofEpochMilli(metadata.getPublishTime())));
        }
        // Decode if needed
        CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
        ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
        // Copy into a heap buffer for output stream compatibility
        ByteBuf data = PooledByteBufAllocator.DEFAULT.heapBuffer(uncompressedPayload.readableBytes(), uncompressedPayload.readableBytes());
        data.writeBytes(uncompressedPayload);
        uncompressedPayload.release();
        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream output) throws IOException, WebApplicationException {
                output.write(data.array(), data.arrayOffset(), data.readableBytes());
                data.release();
            }
        };
        return responseBuilder.entity(stream).build();
    } catch (NullPointerException npe) {
        throw new RestException(Status.NOT_FOUND, "Message not found");
    } catch (Exception exception) {
        log.error("[{}] Failed to get message at position {} from {} {}", clientAppId(), messagePosition, dn, subName, exception);
        throw new RestException(exception);
    } finally {
        if (entry != null) {
            entry.release();
        }
    }
}

Example 5 with MessageMetadata

use of com.yahoo.pulsar.common.api.proto.PulsarApi.MessageMetadata in project pulsar by yahoo.

the class ConsumerImpl method messageReceived.

void messageReceived(MessageIdData messageId, ByteBuf headersAndPayload, ClientCnx cnx) {
    if (log.isDebugEnabled()) {
        log.debug("[{}][{}] Received message: {}/{}", topic, subscription, messageId.getLedgerId(), messageId.getEntryId());
    }
    MessageMetadata msgMetadata = null;
    ByteBuf payload = headersAndPayload;
    if (!verifyChecksum(headersAndPayload, messageId)) {
        // discard message with checksum error
        discardCorruptedMessage(messageId, cnx, ValidationError.ChecksumMismatch);
        return;
    }
    try {
        msgMetadata = Commands.parseMessageMetadata(payload);
    } catch (Throwable t) {
        discardCorruptedMessage(messageId, cnx, ValidationError.ChecksumMismatch);
        return;
    }
    ByteBuf uncompressedPayload = uncompressPayloadIfNeeded(messageId, msgMetadata, payload, cnx);
    if (uncompressedPayload == null) {
        // Message was discarded on decompression error
        return;
    }
    final int numMessages = msgMetadata.getNumMessagesInBatch();
    if (numMessages == 1 && !msgMetadata.hasNumMessagesInBatch()) {
        final MessageImpl message = new MessageImpl(messageId, msgMetadata, uncompressedPayload, getPartitionIndex(), cnx);
        uncompressedPayload.release();
        msgMetadata.recycle();
        try {
            lock.readLock().lock();
            // Enqueue the message so that it can be retrieved when application calls receive()
            // if the conf.getReceiverQueueSize() is 0 then discard message if no one is waiting for it.
            // if asyncReceive is waiting then notify callback without adding to incomingMessages queue
            unAckedMessageTracker.add((MessageIdImpl) message.getMessageId());
            boolean asyncReceivedWaiting = !pendingReceives.isEmpty();
            if ((conf.getReceiverQueueSize() != 0 || waitingOnReceiveForZeroQueueSize) && !asyncReceivedWaiting) {
                incomingMessages.add(message);
            }
            if (asyncReceivedWaiting) {
                notifyPendingReceivedCallback(message, null);
            }
        } finally {
            lock.readLock().unlock();
        }
    } else {
        if (conf.getReceiverQueueSize() == 0) {
            log.warn("Closing consumer [{}]-[{}] due to unsupported received batch-message with zero receiver queue size", subscription, consumerName);
            // close connection
            closeAsync().handle((ok, e) -> {
                // notify callback with failure result
                notifyPendingReceivedCallback(null, new PulsarClientException.InvalidMessageException(format("Unsupported Batch message with 0 size receiver queue for [%s]-[%s] ", subscription, consumerName)));
                return null;
            });
        } else {
            // handle batch message enqueuing; uncompressed payload has all messages in batch
            receiveIndividualMessagesFromBatch(msgMetadata, uncompressedPayload, messageId, cnx);
        }
        uncompressedPayload.release();
        msgMetadata.recycle();
    }
    if (listener != null) {
        // Trigger the notification on the message listener in a separate thread to avoid blocking the networking
        // thread while the message processing happens
        listenerExecutor.execute(() -> {
            for (int i = 0; i < numMessages; i++) {
                try {
                    Message msg = internalReceive(0, TimeUnit.MILLISECONDS);
                    // complete the callback-loop in case queue is cleared up
                    if (msg == null) {
                        if (log.isDebugEnabled()) {
                            log.debug("[{}] [{}] Message has been cleared from the queue", topic, subscription);
                        }
                        break;
                    }
                    try {
                        if (log.isDebugEnabled()) {
                            log.debug("[{}][{}] Calling message listener for message {}", topic, subscription, msg.getMessageId());
                        }
                        listener.received(ConsumerImpl.this, msg);
                    } catch (Throwable t) {
                        log.error("[{}][{}] Message listener error in processing message: {}", topic, subscription, msg.getMessageId(), t);
                    }
                } catch (PulsarClientException e) {
                    log.warn("[{}] [{}] Failed to dequeue the message for listener", topic, subscription, e);
                    return;
                }
            }
        });
    }
}