Examples with SingleMessageMetadata org.apache.pulsar.common.api.proto.SingleMessageMetadata used on opensource projects

Example 1 with SingleMessageMetadata

use of org.apache.pulsar.common.api.proto.SingleMessageMetadata in project pulsar by apache.

the class RawBatchConverter method extractIdsAndKeysAndSize.

public static List<ImmutableTriple<MessageId, String, Integer>> extractIdsAndKeysAndSize(RawMessage msg) throws IOException {
    checkArgument(msg.getMessageIdData().getBatchIndex() == -1);
    ByteBuf payload = msg.getHeadersAndPayload();
    MessageMetadata metadata = Commands.parseMessageMetadata(payload);
    int batchSize = metadata.getNumMessagesInBatch();
    CompressionType compressionType = metadata.getCompression();
    CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType);
    int uncompressedSize = metadata.getUncompressedSize();
    ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize);
    List<ImmutableTriple<MessageId, String, Integer>> idsAndKeysAndSize = new ArrayList<>();
    SingleMessageMetadata smm = new SingleMessageMetadata();
    for (int i = 0; i < batchSize; i++) {
        ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, smm, 0, batchSize);
        MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i);
        if (!smm.isCompactedOut()) {
            idsAndKeysAndSize.add(ImmutableTriple.of(id, smm.hasPartitionKey() ? smm.getPartitionKey() : null, smm.hasPayloadSize() ? smm.getPayloadSize() : 0));
        }
        singleMessagePayload.release();
    }
    uncompressedPayload.release();
    return idsAndKeysAndSize;
}

Example 2 with SingleMessageMetadata

use of org.apache.pulsar.common.api.proto.SingleMessageMetadata in project pulsar by apache.

the class RawBatchConverter method rebatchMessage.

/**
 * Take a batched message and a filter, and returns a message with the only the sub-messages
 * which match the filter. Returns an empty optional if no messages match.
 *
 *  NOTE: this message does not alter the reference count of the RawMessage argument.
 */
public static Optional<RawMessage> rebatchMessage(RawMessage msg, BiPredicate<String, MessageId> filter) throws IOException {
    checkArgument(msg.getMessageIdData().getBatchIndex() == -1);
    ByteBuf payload = msg.getHeadersAndPayload();
    MessageMetadata metadata = Commands.parseMessageMetadata(payload);
    ByteBuf batchBuffer = PulsarByteBufAllocator.DEFAULT.buffer(payload.capacity());
    CompressionType compressionType = metadata.getCompression();
    CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType);
    int uncompressedSize = metadata.getUncompressedSize();
    ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize);
    try {
        int batchSize = metadata.getNumMessagesInBatch();
        int messagesRetained = 0;
        SingleMessageMetadata emptyMetadata = new SingleMessageMetadata().setCompactedOut(true);
        SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata();
        for (int i = 0; i < batchSize; i++) {
            ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadata, 0, batchSize);
            MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i);
            if (!singleMessageMetadata.hasPartitionKey()) {
                messagesRetained++;
                Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadata, singleMessagePayload, batchBuffer);
            } else if (filter.test(singleMessageMetadata.getPartitionKey(), id) && singleMessagePayload.readableBytes() > 0) {
                messagesRetained++;
                Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadata, singleMessagePayload, batchBuffer);
            } else {
                Commands.serializeSingleMessageInBatchWithPayload(emptyMetadata, Unpooled.EMPTY_BUFFER, batchBuffer);
            }
            singleMessagePayload.release();
        }
        if (messagesRetained > 0) {
            int newUncompressedSize = batchBuffer.readableBytes();
            ByteBuf compressedPayload = codec.encode(batchBuffer);
            metadata.setUncompressedSize(newUncompressedSize);
            ByteBuf metadataAndPayload = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, metadata, compressedPayload);
            Optional<RawMessage> result = Optional.of(new RawMessageImpl(msg.getMessageIdData(), metadataAndPayload));
            metadataAndPayload.release();
            compressedPayload.release();
            return result;
        } else {
            return Optional.empty();
        }
    } finally {
        uncompressedPayload.release();
        batchBuffer.release();
    }
}

Example 3 with SingleMessageMetadata

use of org.apache.pulsar.common.api.proto.SingleMessageMetadata in project pulsar by apache.

the class TopicsImpl method getIndividualMsgsFromBatch.

private List<Message<byte[]>> getIndividualMsgsFromBatch(String topic, String msgId, byte[] data, Map<String, String> properties, MessageMetadata msgMetadataBuilder, BrokerEntryMetadata brokerEntryMetadata) {
    List<Message<byte[]>> ret = new ArrayList<>();
    int batchSize = Integer.parseInt(properties.get(BATCH_HEADER));
    ByteBuf buf = Unpooled.wrappedBuffer(data);
    for (int i = 0; i < batchSize; i++) {
        String batchMsgId = msgId + ":" + i;
        SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata();
        try {
            ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(buf, singleMessageMetadata, i, batchSize);
            if (singleMessageMetadata.getPropertiesCount() > 0) {
                for (KeyValue entry : singleMessageMetadata.getPropertiesList()) {
                    properties.put(entry.getKey(), entry.getValue());
                }
            }
            MessageImpl message = new MessageImpl<>(topic, batchMsgId, properties, singleMessagePayload, Schema.BYTES, msgMetadataBuilder);
            if (brokerEntryMetadata != null) {
                message.setBrokerEntryMetadata(brokerEntryMetadata);
            }
            ret.add(message);
        } catch (Exception ex) {
            log.error("Exception occurred while trying to get BatchMsgId: {}", batchMsgId, ex);
        }
    }
    buf.release();
    return ret;
}

Example 4 with SingleMessageMetadata

use of org.apache.pulsar.common.api.proto.SingleMessageMetadata in project pulsar by apache.

the class Commands method serializeSingleMessageInBatchWithPayload.

public static ByteBuf serializeSingleMessageInBatchWithPayload(MessageMetadata msg, ByteBuf payload, ByteBuf batchBuffer) {
    // build single message meta-data
    SingleMessageMetadata smm = LOCAL_SINGLE_MESSAGE_METADATA.get();
    smm.clear();
    if (msg.hasPartitionKey()) {
        smm.setPartitionKey(msg.getPartitionKey());
        smm.setPartitionKeyB64Encoded(msg.isPartitionKeyB64Encoded());
    }
    if (msg.hasOrderingKey()) {
        smm.setOrderingKey(msg.getOrderingKey());
    }
    for (int i = 0; i < msg.getPropertiesCount(); i++) {
        smm.addProperty().setKey(msg.getPropertyAt(i).getKey()).setValue(msg.getPropertyAt(i).getValue());
    }
    if (msg.hasEventTime()) {
        smm.setEventTime(msg.getEventTime());
    }
    if (msg.hasSequenceId()) {
        smm.setSequenceId(msg.getSequenceId());
    }
    if (msg.hasNullValue()) {
        smm.setNullValue(msg.isNullValue());
    }
    if (msg.hasNullPartitionKey()) {
        smm.setNullPartitionKey(msg.isNullPartitionKey());
    }
    return serializeSingleMessageInBatchWithPayload(smm, payload, batchBuffer);
}

Example 5 with SingleMessageMetadata

use of org.apache.pulsar.common.api.proto.SingleMessageMetadata in project pulsar by apache.

the class RawMessageImplTest method testGetProperties.

@Test
public void testGetProperties() {
    ReferenceCountedMessageMetadata refCntMsgMetadata = ReferenceCountedMessageMetadata.get(Mockito.mock(ByteBuf.class));
    SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata();
    singleMessageMetadata.addProperty().setKey(HARD_CODE_KEY).setValue(KEY_VALUE_FIRST);
    singleMessageMetadata.addProperty().setKey(HARD_CODE_KEY).setValue(KEY_VALUE_SECOND);
    singleMessageMetadata.addProperty().setKey(HARD_CODE_KEY_ID).setValue(HARD_CODE_KEY_ID_VALUE);
    RawMessage msg = RawMessageImpl.get(refCntMsgMetadata, singleMessageMetadata, null, 0, 0, 0);
    Map<String, String> properties = msg.getProperties();
    assertEquals(properties.get(HARD_CODE_KEY), KEY_VALUE_SECOND);
    assertEquals(properties.get(HARD_CODE_KEY_ID), HARD_CODE_KEY_ID_VALUE);
    assertEquals(KEY_VALUE_SECOND, properties.get(HARD_CODE_KEY));
    assertEquals(HARD_CODE_KEY_ID_VALUE, properties.get(HARD_CODE_KEY_ID));
}