Examples with LZ4Factory net.jpountz.lz4.LZ4Factory used on opensource projects

Example 1 with LZ4Factory

use of net.jpountz.lz4.LZ4Factory in project osm4j-pbf by topobyte.

the class PbfUtil method initLz4.

private static void initLz4() {
    if (lz4Decompressor == null) {
        LZ4Factory factory = LZ4Factory.fastestInstance();
        lz4Decompressor = factory.fastDecompressor();
    }
}

Example 2 with LZ4Factory

use of net.jpountz.lz4.LZ4Factory in project vespa by vespa-engine.

the class BasicPacket method encodeAndCompressBody.

protected void encodeAndCompressBody(ByteBuffer buffer, int startPosition) {
    int startOfBody = buffer.position();
    encodeBody(buffer);
    setEncodedBody(buffer, startOfBody, buffer.position() - startOfBody);
    length = buffer.position() - startPosition;
    if (compressionLimit != 0 && length - 4 > compressionLimit) {
        byte[] compressedBody;
        compressionType = CompressionType.LZ4;
        LZ4Factory factory = LZ4Factory.fastestInstance();
        LZ4Compressor compressor = factory.fastCompressor();
        compressedBody = compressor.compress(encodedBody);
        log.log(LogLevel.DEBUG, "Uncompressed size: " + encodedBody.length + ", Compressed size: " + compressedBody.length);
        if (compressedBody.length + 4 < encodedBody.length) {
            buffer.position(startPosition);
            // +4 for compressed size
            buffer.putInt(compressedBody.length + startOfBody - startPosition + 4 - 4);
            buffer.putInt(getCompressedCode(compressionType));
            buffer.position(startOfBody);
            buffer.putInt(encodedBody.length);
            buffer.put(compressedBody);
            buffer.limit(buffer.position());
            return;
        }
    }
    // Encoded length 4 less than actual length
    buffer.putInt(startPosition, length - 4);
    buffer.limit(buffer.position());
}

Example 3 with LZ4Factory

use of net.jpountz.lz4.LZ4Factory in project eat by nhnent.

the class ProtoBufPacket method receivePacket.

/**
 * Inspect packet and parse packet
 *  1. Read packet header
 *  2. If size of packet is lack to generated whole packet, it will wait next packet
 *  3. If 2 more packets are exist in the given packet, it will insert it to given queue
 * @param packets Queue of packets
 * @param data Original packet byte
 * @throws Exception Handle exception
 */
@Override
public final void receivePacket(Queue<Pair<String, byte[]>> packets, final byte[] data) throws Exception {
    String receivedPacketName;
    if (buffer.writableBytes() < data.length) {
        int copyCount = buffer.writerIndex() - buffer.readerIndex();
        buffer.resetWriterIndex();
        buffer.writeBytes(buffer, buffer.readerIndex(), copyCount);
        buffer.resetReaderIndex();
        if (buffer.writableBytes() < data.length) {
            int addCapacity = data.length - buffer.writableBytes();
            buffer.capacity(buffer.capacity() + addCapacity);
        }
    }
    buffer.writeBytes(data);
    while (buffer.readableBytes() != 0) {
        int headerSize = buffer.getByte(buffer.readerIndex());
        int streamLen = buffer.readableBytes();
        // If lack of header packet size
        if (streamLen < headerSize + 1) {
            // Add 1 byte for header size.
            break;
        }
        final String baseHeader = ProtobufConfig.obj().getProtobuf().getHeaderPackageClassName() + "$Header";
        final String baseHeaderBuilder = ProtobufConfig.obj().getProtobuf().getHeaderPackageClassName() + "$Header$Builder";
        PacketClassPool classes = PacketClassPool.obj();
        Object builder = classes.callFunction(baseHeader, null, "newBuilder", null);
        try {
            classes.callFunction(baseHeaderBuilder, builder, "mergeFrom", buffer.array(), buffer.readerIndex() + 1, headerSize);
        } catch (Exception e) {
            logger.error("Failed to decode packet.\nheaderSize:{}\nData:{}", headerSize, buffer.array());
            throw e;
        }
        // Check and skip Ping packet
        String msgName = (String) classes.callFunction(baseHeaderBuilder, builder, "getMsgName", null);
        // logger.info("recv Msg Name : " + msgName);
        if (msgName.equals("Ping")) {
            logger.debug("received PING");
            buffer.readerIndex(buffer.readerIndex() + 1 + headerSize);
            continue;
        }
        boolean needToDecompress = false;
        int bodySize;
        int uncompressSize = (int) classes.callFunction(baseHeaderBuilder, builder, "getUncompressSize", null);
        logger.debug("msgName=>{}, uncompressSize=>{}", msgName, uncompressSize);
        bodySize = (int) classes.callFunction(baseHeaderBuilder, builder, "getBodySize", null);
        int packetSize = 1 + headerSize + bodySize;
        if (uncompressSize != 0) {
            // Need to decompress packet
            needToDecompress = true;
        }
        // If lack of packet size
        if (packetSize > streamLen) {
            break;
        }
        byte[] body;
        if (needToDecompress) {
            byte[] compressedBody = new byte[bodySize];
            // begin : code for lz4 decompression
            body = new byte[uncompressSize];
            System.arraycopy(buffer.array(), buffer.readerIndex() + 1 + headerSize, compressedBody, 0, bodySize);
            LZ4Factory factory = LZ4Factory.fastestInstance();
            LZ4FastDecompressor decompressor = factory.fastDecompressor();
            decompressor.decompress(compressedBody, body);
        // end : code for lz4 decompression
        // begin : code for snappy decompression
        // body = Snappy.uncompress(compressedBody);
        // end : code for snappy decompression
        } else {
            body = new byte[bodySize];
            System.arraycopy(buffer.array(), buffer.readerIndex() + 1 + headerSize, body, 0, bodySize);
        }
        if (headerSize != 0) {
            receivedPacketName = (String) classes.callFunction(baseHeaderBuilder, builder, "getMsgName", null);
            packets.add(new Pair<>(receivedPacketName, body));
        }
        buffer.readerIndex(buffer.readerIndex() + packetSize);
    }
}

Example 4 with LZ4Factory

use of net.jpountz.lz4.LZ4Factory in project osm4j-pbf by topobyte.

the class BlockWriter method initLz4.

private void initLz4() {
    if (lz4Compressor == null) {
        LZ4Factory factory = LZ4Factory.fastestInstance();
        lz4Compressor = factory.fastCompressor();
    }
}