Examples with ByteBufCodedOutputStream com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream used on opensource projects

Example 1 with ByteBufCodedOutputStream

use of com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream in project pulsar by yahoo.

the class MessageIdImpl method toByteArray.

// batchIndex is -1 if message is non-batched message and has the batchIndex for a batch message
protected byte[] toByteArray(int batchIndex) {
    MessageIdData.Builder builder = MessageIdData.newBuilder();
    builder.setLedgerId(ledgerId);
    builder.setEntryId(entryId);
    if (partitionIndex >= 0) {
        builder.setPartition(partitionIndex);
    }
    if (batchIndex != -1) {
        builder.setBatchIndex(batchIndex);
    }
    MessageIdData msgId = builder.build();
    int size = msgId.getSerializedSize();
    ByteBuf serialized = Unpooled.buffer(size, size);
    ByteBufCodedOutputStream stream = ByteBufCodedOutputStream.get(serialized);
    try {
        msgId.writeTo(stream);
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    }
    msgId.recycle();
    builder.recycle();
    stream.recycle();
    return serialized.array();
}

Example 2 with ByteBufCodedOutputStream

use of com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream in project pulsar by yahoo.

the class Commands method serializeWithSize.

private static ByteBuf serializeWithSize(BaseCommand.Builder cmdBuilder) {
    // / Wire format
    // [TOTAL_SIZE] [CMD_SIZE][CMD]
    BaseCommand cmd = cmdBuilder.build();
    int cmdSize = cmd.getSerializedSize();
    int totalSize = cmdSize + 4;
    int frameSize = totalSize + 4;
    ByteBuf buf = PooledByteBufAllocator.DEFAULT.buffer(frameSize, frameSize);
    // Prepend 2 lengths to the buffer
    buf.writeInt(totalSize);
    buf.writeInt(cmdSize);
    ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(buf);
    try {
        cmd.writeTo(outStream);
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    } finally {
        cmd.recycle();
        cmdBuilder.recycle();
        outStream.recycle();
    }
    return buf;
}

Example 3 with ByteBufCodedOutputStream

use of com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream in project pulsar by yahoo.

the class Commands method serializeSingleMessageInBatchWithPayload.

public static ByteBuf serializeSingleMessageInBatchWithPayload(PulsarApi.MessageMetadata.Builder msgBuilder, ByteBuf payload, ByteBuf batchBuffer) {
    // build single message meta-data
    PulsarApi.SingleMessageMetadata.Builder singleMessageMetadataBuilder = PulsarApi.SingleMessageMetadata.newBuilder();
    if (msgBuilder.hasPartitionKey()) {
        singleMessageMetadataBuilder = singleMessageMetadataBuilder.setPartitionKey(msgBuilder.getPartitionKey());
    }
    if (!msgBuilder.getPropertiesList().isEmpty()) {
        singleMessageMetadataBuilder = singleMessageMetadataBuilder.addAllProperties(msgBuilder.getPropertiesList());
    }
    int payLoadSize = payload.readableBytes();
    PulsarApi.SingleMessageMetadata singleMessageMetadata = singleMessageMetadataBuilder.setPayloadSize(payLoadSize).build();
    singleMessageMetadataBuilder.recycle();
    // serialize meta-data size, meta-data and payload for single message in batch
    int singleMsgMetadataSize = singleMessageMetadata.getSerializedSize();
    try {
        batchBuffer.writeInt(singleMsgMetadataSize);
        ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(batchBuffer);
        singleMessageMetadata.writeTo(outStream);
        singleMessageMetadata.recycle();
        outStream.recycle();
    } catch (IOException e) {
        throw new RuntimeException(e);
    }
    return batchBuffer.writeBytes(payload);
}

Example 4 with ByteBufCodedOutputStream

use of com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream in project pulsar by yahoo.

the class Commands method serializeCommandSendWithSize.

private static ByteBuf serializeCommandSendWithSize(BaseCommand.Builder cmdBuilder, ChecksumType checksumType, MessageMetadata msgMetadata, ByteBuf payload) {
    // / Wire format
    // [TOTAL_SIZE] [CMD_SIZE][CMD] [MAGIC_NUMBER][CHECKSUM] [METADATA_SIZE][METADATA] [PAYLOAD]
    BaseCommand cmd = cmdBuilder.build();
    int cmdSize = cmd.getSerializedSize();
    int msgMetadataSize = msgMetadata.getSerializedSize();
    int payloadSize = payload.readableBytes();
    int magicAndChecksumLength = ChecksumType.Crc32c.equals(checksumType) ? (2 + 4) : 0;
    boolean includeChecksum = magicAndChecksumLength > 0;
    // cmdLength + cmdSize + magicLength +
    int headerContentSize = 4 + cmdSize + magicAndChecksumLength + 4 + msgMetadataSize;
    // checksumSize + msgMetadataLength +
    // msgMetadataSize
    int totalSize = headerContentSize + payloadSize;
    // totalSize + headerLength
    int headersSize = 4 + headerContentSize;
    int checksumReaderIndex = -1;
    ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize, headersSize);
    // External frame
    headers.writeInt(totalSize);
    try {
        // Write cmd
        headers.writeInt(cmdSize);
        ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
        cmd.writeTo(outStream);
        cmd.recycle();
        cmdBuilder.recycle();
        //Create checksum placeholder
        if (includeChecksum) {
            headers.writeShort(magicCrc32c);
            checksumReaderIndex = headers.writerIndex();
            //skip 4 bytes of checksum
            headers.writerIndex(headers.writerIndex() + checksumSize);
        }
        // Write metadata
        headers.writeInt(msgMetadataSize);
        msgMetadata.writeTo(outStream);
        outStream.recycle();
    } catch (IOException e) {
        // This is in-memory serialization, should not fail
        throw new RuntimeException(e);
    }
    ByteBuf command = DoubleByteBuf.get(headers, payload);
    // write checksum at created checksum-placeholder
    if (includeChecksum) {
        headers.markReaderIndex();
        headers.readerIndex(checksumReaderIndex + checksumSize);
        int metadataChecksum = computeChecksum(headers);
        int computedChecksum = resumeChecksum(metadataChecksum, payload);
        // set computed checksum
        headers.setInt(checksumReaderIndex, computedChecksum);
        headers.resetReaderIndex();
    }
    return command;
}

Example 5 with ByteBufCodedOutputStream

use of com.yahoo.pulsar.common.util.protobuf.ByteBufCodedOutputStream in project pulsar by yahoo.

the class PersistentMessageFinderTest method createMessageWrittenToLedger.

public static byte[] createMessageWrittenToLedger(String msg) throws Exception {
    PulsarApi.MessageMetadata.Builder messageMetadataBuilder = PulsarApi.MessageMetadata.newBuilder();
    messageMetadataBuilder.setPublishTime(System.currentTimeMillis());
    messageMetadataBuilder.setProducerName("createMessageWrittenToLedger");
    messageMetadataBuilder.setSequenceId(1);
    PulsarApi.MessageMetadata messageMetadata = messageMetadataBuilder.build();
    ByteBuf data = UnpooledByteBufAllocator.DEFAULT.heapBuffer().writeBytes(msg.getBytes());
    int msgMetadataSize = messageMetadata.getSerializedSize();
    int payloadSize = data.readableBytes();
    int totalSize = 4 + msgMetadataSize + payloadSize;
    ByteBuf headers = PooledByteBufAllocator.DEFAULT.heapBuffer(totalSize, totalSize);
    ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
    headers.writeInt(msgMetadataSize);
    messageMetadata.writeTo(outStream);
    ByteBuf headersAndPayload = DoubleByteBuf.get(headers, data);
    byte[] byteMessage = headersAndPayload.nioBuffer().array();
    headersAndPayload.release();
    return byteMessage;
}