Examples with MessageMetadata org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata used on opensource projects

Example 1 with MessageMetadata

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata in project incubator-pulsar by apache.

the class ManagedLedgerTest method getMessageWithMetadata.

public ByteBuf getMessageWithMetadata(byte[] data) throws IOException {
    MessageMetadata messageData = MessageMetadata.newBuilder().setPublishTime(System.currentTimeMillis()).setProducerName("prod-name").setSequenceId(0).build();
    ByteBuf payload = Unpooled.wrappedBuffer(data, 0, data.length);
    int msgMetadataSize = messageData.getSerializedSize();
    int headersSize = 4 + msgMetadataSize;
    ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize, headersSize);
    ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
    headers.writeInt(msgMetadataSize);
    messageData.writeTo(outStream);
    outStream.recycle();
    return ByteBufPair.coalesce(ByteBufPair.get(headers, payload));
}

Example 2 with MessageMetadata

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata in project incubator-pulsar by apache.

the class MessageIdTest method testCorruptMessageRemove.

/**
 * Verifies: if message is corrupted before sending to broker and if broker gives checksum error: then 1.
 * Client-Producer recomputes checksum with modified data 2. Retry message-send again 3. Broker verifies checksum 4.
 * client receives send-ack success
 *
 * @throws Exception
 */
@Test
public void testCorruptMessageRemove() throws Exception {
    final String topicName = "persistent://prop/use/ns-abc/retry-topic";
    // 1. producer connect
    ProducerImpl<byte[]> prod = (ProducerImpl<byte[]>) pulsarClient.newProducer().topic(topicName).sendTimeout(10, TimeUnit.MINUTES).create();
    ProducerImpl<byte[]> producer = spy(prod);
    Field producerIdField = ProducerImpl.class.getDeclaredField("producerId");
    producerIdField.setAccessible(true);
    long producerId = (long) producerIdField.get(producer);
    // registered spy ProducerImpl
    producer.cnx().registerProducer(producerId, producer);
    Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
    // 2. Stop the broker, and publishes messages. Messages are accumulated in the producer queue and they're
    // checksums
    // would have already been computed. If we change the message content at that point, it should result in a
    // checksum validation error
    // enable checksum at producer
    stopBroker();
    Message<byte[]> msg = MessageBuilder.create().setContent("message-1".getBytes()).build();
    CompletableFuture<MessageId> future = producer.sendAsync(msg);
    // 3. corrupt the message
    // new content would be 'message-3'
    msg.getData()[msg.getData().length - 1] = '2';
    // 4. Restart the broker to have the messages published
    startBroker();
    try {
        future.get();
        fail("send message should have failed with checksum excetion");
    } catch (Exception e) {
        if (e.getCause() instanceof PulsarClientException.ChecksumException) {
        // ok (callback should get checksum exception as message was modified and corrupt)
        } else {
            fail("Callback should have only failed with ChecksumException", e);
        }
    }
    // 5. Verify
    /**
     * verify: ProducerImpl.verifyLocalBufferIsNotCorrupted() => validates if message is corrupt
     */
    MessageImpl<byte[]> msg2 = (MessageImpl<byte[]>) MessageBuilder.create().setContent("message-1".getBytes()).build();
    ByteBuf payload = msg2.getDataBuffer();
    Builder metadataBuilder = ((MessageImpl<byte[]>) msg).getMessageBuilder();
    MessageMetadata msgMetadata = metadataBuilder.setProducerName("test").setSequenceId(1).setPublishTime(10L).build();
    ByteBufPair cmd = Commands.newSend(producerId, 1, 1, ChecksumType.Crc32c, msgMetadata, payload);
    // (a) create OpSendMsg with message-data : "message-1"
    OpSendMsg op = OpSendMsg.create(((MessageImpl<byte[]>) msg), cmd, 1, null);
    // a.verify: as message is not corrupt: no need to update checksum
    assertTrue(producer.verifyLocalBufferIsNotCorrupted(op));
    // (b) corrupt message
    // new content would be 'message-2'
    msg2.getData()[msg2.getData().length - 1] = '2';
    // b. verify: as message is corrupt: update checksum
    assertFalse(producer.verifyLocalBufferIsNotCorrupted(op));
    assertEquals(producer.getPendingQueueSize(), 0);
    // [2] test-recoverChecksumError functionality
    stopBroker();
    MessageImpl<byte[]> msg1 = (MessageImpl<byte[]>) MessageBuilder.create().setContent("message-1".getBytes()).build();
    future = producer.sendAsync(msg1);
    ClientCnx cnx = spy(new ClientCnx(new ClientConfigurationData(), ((PulsarClientImpl) pulsarClient).eventLoopGroup()));
    String exc = "broker is already stopped";
    // when client-try to recover checksum by resending to broker: throw exception as broker is stopped
    doThrow(new IllegalStateException(exc)).when(cnx).ctx();
    try {
        producer.recoverChecksumError(cnx, 1);
        fail("it should call : resendMessages() => which should throw above mocked exception");
    } catch (IllegalStateException e) {
        assertEquals(exc, e.getMessage());
    }
    producer.close();
    consumer.close();
    // clean reference of mocked producer
    producer = null;
}

Example 3 with MessageMetadata

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata in project incubator-pulsar by apache.

the class RawReaderTest method extractKey.

public static String extractKey(RawMessage m) throws Exception {
    ByteBuf headersAndPayload = m.getHeadersAndPayload();
    MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload);
    return msgMetadata.getPartitionKey();
}

Example 4 with MessageMetadata

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata in project incubator-pulsar by apache.

the class Producer method publishMessage.

public void publishMessage(long producerId, long sequenceId, ByteBuf headersAndPayload, long batchSize) {
    if (isClosed) {
        cnx.ctx().channel().eventLoop().execute(() -> {
            cnx.ctx().writeAndFlush(Commands.newSendError(producerId, sequenceId, ServerError.PersistenceError, "Producer is closed"));
            cnx.completedSendOperation(isNonPersistentTopic);
        });
        return;
    }
    if (!verifyChecksum(headersAndPayload)) {
        cnx.ctx().channel().eventLoop().execute(() -> {
            cnx.ctx().writeAndFlush(Commands.newSendError(producerId, sequenceId, ServerError.ChecksumError, "Checksum failed on the broker"));
            cnx.completedSendOperation(isNonPersistentTopic);
        });
        return;
    }
    if (topic.isEncryptionRequired()) {
        headersAndPayload.markReaderIndex();
        MessageMetadata msgMetadata = Commands.parseMessageMetadata(headersAndPayload);
        headersAndPayload.resetReaderIndex();
        // Check whether the message is encrypted or not
        if (msgMetadata.getEncryptionKeysCount() < 1) {
            log.warn("[{}] Messages must be encrypted", getTopic().getName());
            cnx.ctx().channel().eventLoop().execute(() -> {
                cnx.ctx().writeAndFlush(Commands.newSendError(producerId, sequenceId, ServerError.MetadataError, "Messages must be encrypted"));
                cnx.completedSendOperation(isNonPersistentTopic);
            });
            return;
        }
    }
    startPublishOperation();
    topic.publishMessage(headersAndPayload, MessagePublishContext.get(this, sequenceId, msgIn, headersAndPayload.readableBytes(), batchSize));
}

Example 5 with MessageMetadata

use of org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata in project incubator-pulsar by apache.

the class PersistentTopicsBase method internalPeekNthMessage.

protected Response internalPeekNthMessage(String subName, int messagePosition, boolean authoritative) {
    if (topicName.isGlobal()) {
        validateGlobalNamespaceOwnership(namespaceName);
    }
    PartitionedTopicMetadata partitionMetadata = getPartitionedTopicMetadata(topicName, authoritative);
    if (partitionMetadata.partitions > 0) {
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Peek messages on a partitioned topic is not allowed");
    }
    validateAdminOperationOnTopic(authoritative);
    if (!(getTopicReference(topicName) instanceof PersistentTopic)) {
        log.error("[{}] Not supported operation of non-persistent topic {} {}", clientAppId(), topicName, subName);
        throw new RestException(Status.METHOD_NOT_ALLOWED, "Skip messages on a non-persistent topic is not allowed");
    }
    PersistentTopic topic = (PersistentTopic) getTopicReference(topicName);
    PersistentReplicator repl = null;
    PersistentSubscription sub = null;
    Entry entry = null;
    if (subName.startsWith(topic.replicatorPrefix)) {
        repl = getReplicatorReference(subName, topic);
    } else {
        sub = (PersistentSubscription) getSubscriptionReference(subName, topic);
    }
    try {
        if (subName.startsWith(topic.replicatorPrefix)) {
            entry = repl.peekNthMessage(messagePosition).get();
        } else {
            entry = sub.peekNthMessage(messagePosition).get();
        }
        checkNotNull(entry);
        PositionImpl pos = (PositionImpl) entry.getPosition();
        ByteBuf metadataAndPayload = entry.getDataBuffer();
        // moves the readerIndex to the payload
        MessageMetadata metadata = Commands.parseMessageMetadata(metadataAndPayload);
        ResponseBuilder responseBuilder = Response.ok();
        responseBuilder.header("X-Pulsar-Message-ID", pos.toString());
        for (KeyValue keyValue : metadata.getPropertiesList()) {
            responseBuilder.header("X-Pulsar-PROPERTY-" + keyValue.getKey(), keyValue.getValue());
        }
        if (metadata.hasPublishTime()) {
            responseBuilder.header("X-Pulsar-publish-time", DateFormatter.format(metadata.getPublishTime()));
        }
        if (metadata.hasEventTime()) {
            responseBuilder.header("X-Pulsar-event-time", DateFormatter.format(metadata.getEventTime()));
        }
        if (metadata.hasNumMessagesInBatch()) {
            responseBuilder.header("X-Pulsar-num-batch-message", metadata.getNumMessagesInBatch());
        }
        // Decode if needed
        CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
        ByteBuf uncompressedPayload = codec.decode(metadataAndPayload, metadata.getUncompressedSize());
        // Copy into a heap buffer for output stream compatibility
        ByteBuf data = PooledByteBufAllocator.DEFAULT.heapBuffer(uncompressedPayload.readableBytes(), uncompressedPayload.readableBytes());
        data.writeBytes(uncompressedPayload);
        uncompressedPayload.release();
        StreamingOutput stream = new StreamingOutput() {

            @Override
            public void write(OutputStream output) throws IOException, WebApplicationException {
                output.write(data.array(), data.arrayOffset(), data.readableBytes());
                data.release();
            }
        };
        return responseBuilder.entity(stream).build();
    } catch (NullPointerException npe) {
        throw new RestException(Status.NOT_FOUND, "Message not found");
    } catch (Exception exception) {
        log.error("[{}] Failed to get message at position {} from {} {}", clientAppId(), messagePosition, topicName, subName, exception);
        throw new RestException(exception);
    } finally {
        if (entry != null) {
            entry.release();
        }
    }
}