Examples with CompressionType org.apache.pulsar.common.api.proto.CompressionType used on opensource projects

Example 1 with CompressionType

use of org.apache.pulsar.common.api.proto.CompressionType in project pulsar by apache.

the class RawBatchConverter method extractIdsAndKeysAndSize.

public static List<ImmutableTriple<MessageId, String, Integer>> extractIdsAndKeysAndSize(RawMessage msg) throws IOException {
    checkArgument(msg.getMessageIdData().getBatchIndex() == -1);
    ByteBuf payload = msg.getHeadersAndPayload();
    MessageMetadata metadata = Commands.parseMessageMetadata(payload);
    int batchSize = metadata.getNumMessagesInBatch();
    CompressionType compressionType = metadata.getCompression();
    CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType);
    int uncompressedSize = metadata.getUncompressedSize();
    ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize);
    List<ImmutableTriple<MessageId, String, Integer>> idsAndKeysAndSize = new ArrayList<>();
    SingleMessageMetadata smm = new SingleMessageMetadata();
    for (int i = 0; i < batchSize; i++) {
        ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, smm, 0, batchSize);
        MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i);
        if (!smm.isCompactedOut()) {
            idsAndKeysAndSize.add(ImmutableTriple.of(id, smm.hasPartitionKey() ? smm.getPartitionKey() : null, smm.hasPayloadSize() ? smm.getPayloadSize() : 0));
        }
        singleMessagePayload.release();
    }
    uncompressedPayload.release();
    return idsAndKeysAndSize;
}

Example 2 with CompressionType

use of org.apache.pulsar.common.api.proto.CompressionType in project pulsar by apache.

the class RawBatchConverter method rebatchMessage.

/**
 * Take a batched message and a filter, and returns a message with the only the sub-messages
 * which match the filter. Returns an empty optional if no messages match.
 *
 *  NOTE: this message does not alter the reference count of the RawMessage argument.
 */
public static Optional<RawMessage> rebatchMessage(RawMessage msg, BiPredicate<String, MessageId> filter) throws IOException {
    checkArgument(msg.getMessageIdData().getBatchIndex() == -1);
    ByteBuf payload = msg.getHeadersAndPayload();
    MessageMetadata metadata = Commands.parseMessageMetadata(payload);
    ByteBuf batchBuffer = PulsarByteBufAllocator.DEFAULT.buffer(payload.capacity());
    CompressionType compressionType = metadata.getCompression();
    CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType);
    int uncompressedSize = metadata.getUncompressedSize();
    ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize);
    try {
        int batchSize = metadata.getNumMessagesInBatch();
        int messagesRetained = 0;
        SingleMessageMetadata emptyMetadata = new SingleMessageMetadata().setCompactedOut(true);
        SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata();
        for (int i = 0; i < batchSize; i++) {
            ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadata, 0, batchSize);
            MessageId id = new BatchMessageIdImpl(msg.getMessageIdData().getLedgerId(), msg.getMessageIdData().getEntryId(), msg.getMessageIdData().getPartition(), i);
            if (!singleMessageMetadata.hasPartitionKey()) {
                messagesRetained++;
                Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadata, singleMessagePayload, batchBuffer);
            } else if (filter.test(singleMessageMetadata.getPartitionKey(), id) && singleMessagePayload.readableBytes() > 0) {
                messagesRetained++;
                Commands.serializeSingleMessageInBatchWithPayload(singleMessageMetadata, singleMessagePayload, batchBuffer);
            } else {
                Commands.serializeSingleMessageInBatchWithPayload(emptyMetadata, Unpooled.EMPTY_BUFFER, batchBuffer);
            }
            singleMessagePayload.release();
        }
        if (messagesRetained > 0) {
            int newUncompressedSize = batchBuffer.readableBytes();
            ByteBuf compressedPayload = codec.encode(batchBuffer);
            metadata.setUncompressedSize(newUncompressedSize);
            ByteBuf metadataAndPayload = Commands.serializeMetadataAndPayload(Commands.ChecksumType.Crc32c, metadata, compressedPayload);
            Optional<RawMessage> result = Optional.of(new RawMessageImpl(msg.getMessageIdData(), metadataAndPayload));
            metadataAndPayload.release();
            compressedPayload.release();
            return result;
        } else {
            return Optional.empty();
        }
    } finally {
        uncompressedPayload.release();
        batchBuffer.release();
    }
}

Example 3 with CompressionType

use of org.apache.pulsar.common.api.proto.CompressionType in project pulsar by apache.

the class ConsumerImpl method uncompressPayloadIfNeeded.

private ByteBuf uncompressPayloadIfNeeded(MessageIdData messageId, MessageMetadata msgMetadata, ByteBuf payload, ClientCnx currentCnx, boolean checkMaxMessageSize) {
    CompressionType compressionType = msgMetadata.getCompression();
    CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(compressionType);
    int uncompressedSize = msgMetadata.getUncompressedSize();
    int payloadSize = payload.readableBytes();
    if (checkMaxMessageSize && payloadSize > ClientCnx.getMaxMessageSize()) {
        // payload size is itself corrupted since it cannot be bigger than the MaxMessageSize
        log.error("[{}][{}] Got corrupted payload message size {} at {}", topic, subscription, payloadSize, messageId);
        discardCorruptedMessage(messageId, currentCnx, ValidationError.UncompressedSizeCorruption);
        return null;
    }
    try {
        ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize);
        return uncompressedPayload;
    } catch (IOException e) {
        log.error("[{}][{}] Failed to decompress message with {} at {}: {}", topic, subscription, compressionType, messageId, e.getMessage(), e);
        discardCorruptedMessage(messageId, currentCnx, ValidationError.DecompressionError);
        return null;
    }
}