Examples with CompressorStream org.apache.hadoop.io.compress.CompressorStream used on opensource projects

Example 1 with CompressorStream

use of org.apache.hadoop.io.compress.CompressorStream in project hadoop by apache.

the class TestZStandardCompressorDecompressor method compressDecompressLoop.

private void compressDecompressLoop(int rawDataSize) throws IOException {
    byte[] rawData = null;
    rawData = generate(rawDataSize);
    ByteArrayOutputStream baos = new ByteArrayOutputStream(rawDataSize + 12);
    CompressionOutputStream deflateFilter = new CompressorStream(baos, new ZStandardCompressor(), 4096);
    DataOutputStream deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
    deflateOut.write(rawData, 0, rawData.length);
    deflateOut.flush();
    deflateFilter.finish();
    byte[] compressedResult = baos.toByteArray();
    int compressedSize = compressedResult.length;
    ZStandardDecompressor.ZStandardDirectDecompressor decompressor = new ZStandardDecompressor.ZStandardDirectDecompressor(4096);
    ByteBuffer inBuf = ByteBuffer.allocateDirect(compressedSize);
    ByteBuffer outBuf = ByteBuffer.allocateDirect(8096);
    inBuf.put(compressedResult, 0, compressedSize);
    inBuf.flip();
    ByteBuffer expected = ByteBuffer.wrap(rawData);
    outBuf.clear();
    while (!decompressor.finished()) {
        decompressor.decompress(inBuf, outBuf);
        if (outBuf.remaining() == 0) {
            outBuf.flip();
            while (outBuf.remaining() > 0) {
                assertEquals(expected.get(), outBuf.get());
            }
            outBuf.clear();
        }
    }
    outBuf.flip();
    while (outBuf.remaining() > 0) {
        assertEquals(expected.get(), outBuf.get());
    }
    outBuf.clear();
    assertEquals(0, expected.remaining());
}

Example 2 with CompressorStream

use of org.apache.hadoop.io.compress.CompressorStream in project hadoop by apache.

the class TestZStandardCompressorDecompressor method testCompressorDecompressorLogicWithCompressionStreams.

// test compress/decompress process through
// CompressionOutputStream/CompressionInputStream api
@Test
public void testCompressorDecompressorLogicWithCompressionStreams() throws Exception {
    DataOutputStream deflateOut = null;
    DataInputStream inflateIn = null;
    int byteSize = 1024 * 100;
    byte[] bytes = generate(byteSize);
    int bufferSize = IO_FILE_BUFFER_SIZE_DEFAULT;
    try {
        DataOutputBuffer compressedDataBuffer = new DataOutputBuffer();
        CompressionOutputStream deflateFilter = new CompressorStream(compressedDataBuffer, new ZStandardCompressor(), bufferSize);
        deflateOut = new DataOutputStream(new BufferedOutputStream(deflateFilter));
        deflateOut.write(bytes, 0, bytes.length);
        deflateOut.flush();
        deflateFilter.finish();
        DataInputBuffer deCompressedDataBuffer = new DataInputBuffer();
        deCompressedDataBuffer.reset(compressedDataBuffer.getData(), 0, compressedDataBuffer.getLength());
        CompressionInputStream inflateFilter = new DecompressorStream(deCompressedDataBuffer, new ZStandardDecompressor(bufferSize), bufferSize);
        inflateIn = new DataInputStream(new BufferedInputStream(inflateFilter));
        byte[] result = new byte[byteSize];
        inflateIn.read(result);
        assertArrayEquals("original array not equals compress/decompressed array", result, bytes);
    } finally {
        IOUtils.closeQuietly(deflateOut);
        IOUtils.closeQuietly(inflateIn);
    }
}