Examples with DirectBufferOutputStream io.streamnative.pulsar.handlers.kop.format.DirectBufferOutputStream used on opensource projects

Example 1 with DirectBufferOutputStream

use of io.streamnative.pulsar.handlers.kop.format.DirectBufferOutputStream in project starlight-for-kafka by datastax.

the class ByteBufUtils method decodePulsarEntryToKafkaRecords.

public static DecodeResult decodePulsarEntryToKafkaRecords(final MessageMetadata metadata, final ByteBuf payload, final long baseOffset, final byte magic) throws IOException {
    if (metadata.hasMarkerType()) {
        ControlRecordType controlRecordType;
        switch(metadata.getMarkerType()) {
            case MarkerType.TXN_COMMIT_VALUE:
                controlRecordType = ControlRecordType.COMMIT;
                break;
            case MarkerType.TXN_ABORT_VALUE:
                controlRecordType = ControlRecordType.ABORT;
                break;
            default:
                controlRecordType = ControlRecordType.UNKNOWN;
                break;
        }
        return DecodeResult.get(MemoryRecords.withEndTransactionMarker(baseOffset, metadata.getPublishTime(), 0, metadata.getTxnidMostBits(), (short) metadata.getTxnidLeastBits(), new EndTransactionMarker(controlRecordType, 0)));
    }
    long startConversionNanos = MathUtils.nowInNano();
    final int uncompressedSize = metadata.getUncompressedSize();
    final CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
    final ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize);
    final DirectBufferOutputStream directBufferOutputStream = new DirectBufferOutputStream(DEFAULT_BUFFER_SIZE);
    final MemoryRecordsBuilder builder = new MemoryRecordsBuilder(directBufferOutputStream, magic, CompressionType.NONE, TimestampType.CREATE_TIME, baseOffset, metadata.getPublishTime(), RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, metadata.hasTxnidMostBits() && metadata.hasTxnidLeastBits(), false, RecordBatch.NO_PARTITION_LEADER_EPOCH, MAX_RECORDS_BUFFER_SIZE);
    if (metadata.hasTxnidMostBits()) {
        builder.setProducerState(metadata.getTxnidMostBits(), (short) metadata.getTxnidLeastBits(), 0, true);
    }
    int conversionCount = 0;
    if (metadata.hasNumMessagesInBatch()) {
        final int numMessages = metadata.getNumMessagesInBatch();
        conversionCount += numMessages;
        for (int i = 0; i < numMessages; i++) {
            final SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata();
            final ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadata, i, numMessages);
            final long timestamp = (metadata.getEventTime() > 0) ? metadata.getEventTime() : metadata.getPublishTime();
            final ByteBuffer value = singleMessageMetadata.isNullValue() ? null : getNioBuffer(singleMessagePayload);
            if (magic >= RecordBatch.MAGIC_VALUE_V2) {
                final Header[] headers = getHeadersFromMetadata(singleMessageMetadata.getPropertiesList());
                builder.appendWithOffset(baseOffset + i, timestamp, getKeyByteBuffer(singleMessageMetadata), value, headers);
            } else {
                // record less than magic=2, no header attribute
                builder.appendWithOffset(baseOffset + i, timestamp, getKeyByteBuffer(singleMessageMetadata), value);
            }
            singleMessagePayload.release();
        }
    } else {
        conversionCount += 1;
        final long timestamp = (metadata.getEventTime() > 0) ? metadata.getEventTime() : metadata.getPublishTime();
        if (magic >= RecordBatch.MAGIC_VALUE_V2) {
            final Header[] headers = getHeadersFromMetadata(metadata.getPropertiesList());
            builder.appendWithOffset(baseOffset, timestamp, getKeyByteBuffer(metadata), getNioBuffer(uncompressedPayload), headers);
        } else {
            builder.appendWithOffset(baseOffset, timestamp, getKeyByteBuffer(metadata), getNioBuffer(uncompressedPayload));
        }
    }
    final MemoryRecords records = builder.build();
    uncompressedPayload.release();
    return DecodeResult.get(records, directBufferOutputStream.getByteBuf(), conversionCount, MathUtils.elapsedNanos(startConversionNanos));
}

Example 2 with DirectBufferOutputStream

use of io.streamnative.pulsar.handlers.kop.format.DirectBufferOutputStream in project kop by streamnative.

the class ByteBufUtils method decodePulsarEntryToKafkaRecords.

public static DecodeResult decodePulsarEntryToKafkaRecords(final MessageMetadata metadata, final ByteBuf payload, final long baseOffset, final byte magic) throws IOException {
    if (metadata.hasMarkerType()) {
        ControlRecordType controlRecordType;
        switch(metadata.getMarkerType()) {
            case MarkerType.TXN_COMMIT_VALUE:
                controlRecordType = ControlRecordType.COMMIT;
                break;
            case MarkerType.TXN_ABORT_VALUE:
                controlRecordType = ControlRecordType.ABORT;
                break;
            default:
                controlRecordType = ControlRecordType.UNKNOWN;
                break;
        }
        return DecodeResult.get(MemoryRecords.withEndTransactionMarker(baseOffset, metadata.getPublishTime(), 0, metadata.getTxnidMostBits(), (short) metadata.getTxnidLeastBits(), new EndTransactionMarker(controlRecordType, 0)));
    }
    long startConversionNanos = MathUtils.nowInNano();
    final int uncompressedSize = metadata.getUncompressedSize();
    final CompressionCodec codec = CompressionCodecProvider.getCompressionCodec(metadata.getCompression());
    final ByteBuf uncompressedPayload = codec.decode(payload, uncompressedSize);
    final DirectBufferOutputStream directBufferOutputStream = new DirectBufferOutputStream(DEFAULT_BUFFER_SIZE);
    final MemoryRecordsBuilder builder = new MemoryRecordsBuilder(directBufferOutputStream, magic, CompressionType.NONE, TimestampType.CREATE_TIME, baseOffset, metadata.getPublishTime(), RecordBatch.NO_PRODUCER_ID, RecordBatch.NO_PRODUCER_EPOCH, RecordBatch.NO_SEQUENCE, metadata.hasTxnidMostBits() && metadata.hasTxnidLeastBits(), false, RecordBatch.NO_PARTITION_LEADER_EPOCH, MAX_RECORDS_BUFFER_SIZE);
    if (metadata.hasTxnidMostBits()) {
        builder.setProducerState(metadata.getTxnidMostBits(), (short) metadata.getTxnidLeastBits(), 0, true);
    }
    int conversionCount = 0;
    if (metadata.hasNumMessagesInBatch()) {
        final int numMessages = metadata.getNumMessagesInBatch();
        conversionCount += numMessages;
        for (int i = 0; i < numMessages; i++) {
            final SingleMessageMetadata singleMessageMetadata = new SingleMessageMetadata();
            final ByteBuf singleMessagePayload = Commands.deSerializeSingleMessageInBatch(uncompressedPayload, singleMessageMetadata, i, numMessages);
            final long timestamp = (metadata.getEventTime() > 0) ? metadata.getEventTime() : metadata.getPublishTime();
            final ByteBuffer value = singleMessageMetadata.isNullValue() ? null : getNioBuffer(singleMessagePayload);
            if (magic >= RecordBatch.MAGIC_VALUE_V2) {
                final Header[] headers = getHeadersFromMetadata(singleMessageMetadata.getPropertiesList());
                builder.appendWithOffset(baseOffset + i, timestamp, getKeyByteBuffer(singleMessageMetadata), value, headers);
            } else {
                // record less than magic=2, no header attribute
                builder.appendWithOffset(baseOffset + i, timestamp, getKeyByteBuffer(singleMessageMetadata), value);
            }
            singleMessagePayload.release();
        }
    } else {
        conversionCount += 1;
        final long timestamp = (metadata.getEventTime() > 0) ? metadata.getEventTime() : metadata.getPublishTime();
        if (magic >= RecordBatch.MAGIC_VALUE_V2) {
            final Header[] headers = getHeadersFromMetadata(metadata.getPropertiesList());
            builder.appendWithOffset(baseOffset, timestamp, getKeyByteBuffer(metadata), getNioBuffer(uncompressedPayload), headers);
        } else {
            builder.appendWithOffset(baseOffset, timestamp, getKeyByteBuffer(metadata), getNioBuffer(uncompressedPayload));
        }
    }
    final MemoryRecords records = builder.build();
    uncompressedPayload.release();
    return DecodeResult.get(records, directBufferOutputStream.getByteBuf(), conversionCount, MathUtils.elapsedNanos(startConversionNanos));
}