Examples with ByteBufCodedOutputStream org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream used on opensource projects

Example 1 with ByteBufCodedOutputStream

use of org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream in project incubator-pulsar by apache.

the class ManagedLedgerTest method getMessageWithMetadata.

public ByteBuf getMessageWithMetadata(byte[] data) throws IOException {
    MessageMetadata messageData = MessageMetadata.newBuilder().setPublishTime(System.currentTimeMillis()).setProducerName("prod-name").setSequenceId(0).build();
    ByteBuf payload = Unpooled.wrappedBuffer(data, 0, data.length);
    int msgMetadataSize = messageData.getSerializedSize();
    int headersSize = 4 + msgMetadataSize;
    ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize, headersSize);
    ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
    headers.writeInt(msgMetadataSize);
    messageData.writeTo(outStream);
    outStream.recycle();
    return ByteBufPair.coalesce(ByteBufPair.get(headers, payload));
}

Example 2 with ByteBufCodedOutputStream

use of org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream in project incubator-pulsar by apache.

the class Commands method serializeMetadataAndPayload.

public static ByteBuf serializeMetadataAndPayload(ChecksumType checksumType, MessageMetadata msgMetadata, ByteBuf payload) {
    // / Wire format
    // [MAGIC_NUMBER][CHECKSUM] [METADATA_SIZE][METADATA] [PAYLOAD]
    int msgMetadataSize = msgMetadata.getSerializedSize();
    int payloadSize = payload.readableBytes();
    int magicAndChecksumLength = ChecksumType.Crc32c.equals(checksumType) ? (2 + 4) : 0;
    boolean includeChecksum = magicAndChecksumLength > 0;
    // magicLength +
    int headerContentSize = magicAndChecksumLength + 4 + msgMetadataSize;
    // checksumSize + msgMetadataLength +
    // msgMetadataSize
    int checksumReaderIndex = -1;
    int totalSize = headerContentSize + payloadSize;
    ByteBuf metadataAndPayload = PooledByteBufAllocator.DEFAULT.buffer(totalSize, totalSize);
    try {
        ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(metadataAndPayload);
        // Create checksum placeholder
        if (includeChecksum) {
            metadataAndPayload.writeShort(magicCrc32c);
            checksumReaderIndex = metadataAndPayload.writerIndex();
            metadataAndPayload.writerIndex(metadataAndPayload.writerIndex() + // skip 4 bytes of checksum
            checksumSize);
        }
        // Write metadata
        metadataAndPayload.writeInt(msgMetadataSize);
        msgMetadata.writeTo(outStream);
        outStream.recycle();
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    }
    // write checksum at created checksum-placeholder
    if (includeChecksum) {
        metadataAndPayload.markReaderIndex();
        metadataAndPayload.readerIndex(checksumReaderIndex + checksumSize);
        int metadataChecksum = computeChecksum(metadataAndPayload);
        int computedChecksum = resumeChecksum(metadataChecksum, payload);
        // set computed checksum
        metadataAndPayload.setInt(checksumReaderIndex, computedChecksum);
        metadataAndPayload.resetReaderIndex();
    }
    metadataAndPayload.writeBytes(payload);
    return metadataAndPayload;
}

Example 3 with ByteBufCodedOutputStream

use of org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream in project incubator-pulsar by apache.

the class Commands method serializeCommandMessageWithSize.

private static ByteBufPair serializeCommandMessageWithSize(BaseCommand cmd, ByteBuf metadataAndPayload) {
    // / Wire format
    // [TOTAL_SIZE] [CMD_SIZE][CMD] [MAGIC_NUMBER][CHECKSUM] [METADATA_SIZE][METADATA] [PAYLOAD]
    // 
    // metadataAndPayload contains from magic-number to the payload included
    int cmdSize = cmd.getSerializedSize();
    int totalSize = 4 + cmdSize + metadataAndPayload.readableBytes();
    int headersSize = 4 + 4 + cmdSize;
    ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize);
    // External frame
    headers.writeInt(totalSize);
    try {
        // Write cmd
        headers.writeInt(cmdSize);
        ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
        cmd.writeTo(outStream);
        outStream.recycle();
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    }
    return (ByteBufPair) ByteBufPair.get(headers, metadataAndPayload);
}

Example 4 with ByteBufCodedOutputStream

use of org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream in project incubator-pulsar by apache.

the class Commands method serializeWithSize.

@VisibleForTesting
public static ByteBuf serializeWithSize(BaseCommand.Builder cmdBuilder) {
    // / Wire format
    // [TOTAL_SIZE] [CMD_SIZE][CMD]
    BaseCommand cmd = cmdBuilder.build();
    int cmdSize = cmd.getSerializedSize();
    int totalSize = cmdSize + 4;
    int frameSize = totalSize + 4;
    ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(frameSize, frameSize);
    // Prepend 2 lengths to the buffer
    buf.writeInt(totalSize);
    buf.writeInt(cmdSize);
    ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(buf);
    try {
        cmd.writeTo(outStream);
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    } finally {
        cmd.recycle();
        cmdBuilder.recycle();
        outStream.recycle();
    }
    return buf;
}

Example 5 with ByteBufCodedOutputStream

use of org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream in project incubator-pulsar by apache.

the class Commands method serializeCommandSendWithSize.

private static ByteBufPair serializeCommandSendWithSize(BaseCommand.Builder cmdBuilder, ChecksumType checksumType, MessageMetadata msgMetadata, ByteBuf payload) {
    // / Wire format
    // [TOTAL_SIZE] [CMD_SIZE][CMD] [MAGIC_NUMBER][CHECKSUM] [METADATA_SIZE][METADATA] [PAYLOAD]
    BaseCommand cmd = cmdBuilder.build();
    int cmdSize = cmd.getSerializedSize();
    int msgMetadataSize = msgMetadata.getSerializedSize();
    int payloadSize = payload.readableBytes();
    int magicAndChecksumLength = ChecksumType.Crc32c.equals(checksumType) ? (2 + 4) : 0;
    boolean includeChecksum = magicAndChecksumLength > 0;
    // cmdLength + cmdSize + magicLength +
    int headerContentSize = 4 + cmdSize + magicAndChecksumLength + 4 + msgMetadataSize;
    // checksumSize + msgMetadataLength +
    // msgMetadataSize
    int totalSize = headerContentSize + payloadSize;
    // totalSize + headerLength
    int headersSize = 4 + headerContentSize;
    int checksumReaderIndex = -1;
    ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize, headersSize);
    // External frame
    headers.writeInt(totalSize);
    try {
        // Write cmd
        headers.writeInt(cmdSize);
        ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
        cmd.writeTo(outStream);
        cmd.recycle();
        cmdBuilder.recycle();
        // Create checksum placeholder
        if (includeChecksum) {
            headers.writeShort(magicCrc32c);
            checksumReaderIndex = headers.writerIndex();
            // skip 4 bytes of checksum
            headers.writerIndex(headers.writerIndex() + checksumSize);
        }
        // Write metadata
        headers.writeInt(msgMetadataSize);
        msgMetadata.writeTo(outStream);
        outStream.recycle();
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    }
    ByteBufPair command = ByteBufPair.get(headers, payload);
    // write checksum at created checksum-placeholder
    if (includeChecksum) {
        headers.markReaderIndex();
        headers.readerIndex(checksumReaderIndex + checksumSize);
        int metadataChecksum = computeChecksum(headers);
        int computedChecksum = resumeChecksum(metadataChecksum, payload);
        // set computed checksum
        headers.setInt(checksumReaderIndex, computedChecksum);
        headers.resetReaderIndex();
    }
    return command;
}