Examples with ZstdJniCompressor com.facebook.presto.orc.zstd.ZstdJniCompressor used on opensource projects

Example 1 with ZstdJniCompressor

use of com.facebook.presto.orc.zstd.ZstdJniCompressor in project presto by prestodb.

the class TestZstdJniDecompression method testDecompression.

@Test(dataProvider = "zstdCompressionLevels")
public void testDecompression(OptionalInt level) throws OrcCorruptionException {
    byte[] sourceBytes = generateRandomBytes();
    byte[] compressedBytes = new byte[1024 * 1024];
    ZstdJniCompressor compressor = new ZstdJniCompressor(level);
    int size = compressor.compress(sourceBytes, 0, sourceBytes.length, compressedBytes, 0, compressedBytes.length);
    byte[] output = new byte[sourceBytes.length];
    int outputSize = decompressor.decompress(compressedBytes, 0, size, new OrcDecompressor.OutputBuffer() {

        @Override
        public byte[] initialize(int size) {
            return output;
        }

        @Override
        public byte[] grow(int size) {
            throw new RuntimeException();
        }
    });
    Assert.assertEquals(outputSize, sourceBytes.length);
    Assert.assertEquals(output, sourceBytes);
}

Example 2 with ZstdJniCompressor

use of com.facebook.presto.orc.zstd.ZstdJniCompressor in project presto by prestodb.

the class PageFileWriterFactory method createPagesSerdeForPageFile.

public static PagesSerde createPagesSerdeForPageFile(BlockEncodingSerde blockEncodingSerde, Optional<HiveCompressionCodec> compressionCodec) {
    if (!compressionCodec.isPresent()) {
        return new PagesSerde(blockEncodingSerde, Optional.empty(), Optional.empty(), Optional.empty());
    }
    PageCompressor pageCompressor = null;
    PageDecompressor pageDecompressor = null;
    switch(compressionCodec.get()) {
        case NONE:
            break;
        case SNAPPY:
            pageCompressor = new AirliftCompressorAdapter(new SnappyCompressor());
            pageDecompressor = new AirliftDecompressorAdapter(new SnappyDecompressor());
            break;
        case LZ4:
            pageCompressor = new AirliftCompressorAdapter(new Lz4Compressor());
            pageDecompressor = new AirliftDecompressorAdapter(new Lz4Decompressor());
            break;
        case GZIP:
            pageCompressor = new AirliftCompressorAdapter(new DeflateCompressor(OptionalInt.empty()));
            pageDecompressor = new AirliftDecompressorAdapter(new InflateDecompressor());
            break;
        case ZSTD:
            pageCompressor = new AirliftCompressorAdapter(new ZstdJniCompressor(OptionalInt.empty()));
            pageDecompressor = new AirliftDecompressorAdapter(new ZstdJniDecompressor());
            break;
        default:
            throw new PrestoException(GENERIC_INTERNAL_ERROR, format("%s compression is not supported for %s", compressionCodec.get().name(), PAGEFILE.getOutputFormat()));
    }
    return new PagesSerde(blockEncodingSerde, Optional.ofNullable(pageCompressor), Optional.ofNullable(pageDecompressor), Optional.empty());
}