Examples with ZStandardCodec org.apache.hadoop.io.compress.ZStandardCodec used on opensource projects

Example 1 with ZStandardCodec

use of org.apache.hadoop.io.compress.ZStandardCodec in project hadoop by apache.

the class TestZStandardCompressorDecompressor method testReadingWithAStream.

@Test
public void testReadingWithAStream() throws Exception {
    FileInputStream inputStream = FileUtils.openInputStream(compressedFile);
    ZStandardCodec codec = new ZStandardCodec();
    codec.setConf(CONFIGURATION);
    Decompressor decompressor = codec.createDecompressor();
    CompressionInputStream cis = codec.createInputStream(inputStream, decompressor);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    byte[] resultOfDecompression;
    try {
        byte[] buffer = new byte[100];
        int n;
        while ((n = cis.read(buffer, 0, buffer.length)) != -1) {
            baos.write(buffer, 0, n);
        }
        resultOfDecompression = baos.toByteArray();
    } finally {
        IOUtils.closeQuietly(baos);
        IOUtils.closeQuietly(cis);
    }
    byte[] expected = FileUtils.readFileToByteArray(uncompressedFile);
    assertEquals(bytesToHex(expected), bytesToHex(resultOfDecompression));
}

Example 2 with ZStandardCodec

use of org.apache.hadoop.io.compress.ZStandardCodec in project hadoop by apache.

the class TestZStandardCompressorDecompressor method testDecompressingOutput.

@Test
public void testDecompressingOutput() throws Exception {
    byte[] expectedDecompressedResult = FileUtils.readFileToByteArray(uncompressedFile);
    ZStandardCodec codec = new ZStandardCodec();
    codec.setConf(CONFIGURATION);
    CompressionInputStream inputStream = codec.createInputStream(FileUtils.openInputStream(compressedFile), codec.createDecompressor());
    byte[] toDecompress = new byte[100];
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    byte[] decompressedResult;
    int totalFileSize = 0;
    int result = toDecompress.length;
    try {
        while ((result = inputStream.read(toDecompress, 0, result)) != -1) {
            baos.write(toDecompress, 0, result);
            totalFileSize += result;
        }
        decompressedResult = baos.toByteArray();
    } finally {
        IOUtils.closeQuietly(baos);
    }
    assertEquals(decompressedResult.length, totalFileSize);
    assertEquals(bytesToHex(expectedDecompressedResult), bytesToHex(decompressedResult));
}

Example 3 with ZStandardCodec

use of org.apache.hadoop.io.compress.ZStandardCodec in project hadoop by apache.

the class TestZStandardCompressorDecompressor method testCompressionCompressesCorrectly.

@Test
public void testCompressionCompressesCorrectly() throws Exception {
    int uncompressedSize = (int) FileUtils.sizeOf(uncompressedFile);
    byte[] bytes = FileUtils.readFileToByteArray(uncompressedFile);
    assertEquals(uncompressedSize, bytes.length);
    Configuration conf = new Configuration();
    ZStandardCodec codec = new ZStandardCodec();
    codec.setConf(conf);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    Compressor compressor = codec.createCompressor();
    CompressionOutputStream outputStream = codec.createOutputStream(baos, compressor);
    for (byte aByte : bytes) {
        outputStream.write(aByte);
    }
    outputStream.finish();
    outputStream.close();
    assertEquals(uncompressedSize, compressor.getBytesRead());
    assertTrue(compressor.finished());
    // just make sure we can decompress the file
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    Decompressor decompressor = codec.createDecompressor();
    CompressionInputStream inputStream = codec.createInputStream(bais, decompressor);
    byte[] buffer = new byte[100];
    int n = buffer.length;
    while ((n = inputStream.read(buffer, 0, n)) != -1) {
        byteArrayOutputStream.write(buffer, 0, n);
    }
    assertArrayEquals(bytes, byteArrayOutputStream.toByteArray());
}

Example 4 with ZStandardCodec

use of org.apache.hadoop.io.compress.ZStandardCodec in project hadoop by apache.

the class TestZStandardCompressorDecompressor method testCompressingWithOneByteOutputBuffer.

@Test
public void testCompressingWithOneByteOutputBuffer() throws Exception {
    int uncompressedSize = (int) FileUtils.sizeOf(uncompressedFile);
    byte[] bytes = FileUtils.readFileToByteArray(uncompressedFile);
    assertEquals(uncompressedSize, bytes.length);
    Configuration conf = new Configuration();
    ZStandardCodec codec = new ZStandardCodec();
    codec.setConf(conf);
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    Compressor compressor = new ZStandardCompressor(3, IO_FILE_BUFFER_SIZE_DEFAULT, 1);
    CompressionOutputStream outputStream = codec.createOutputStream(baos, compressor);
    for (byte aByte : bytes) {
        outputStream.write(aByte);
    }
    outputStream.finish();
    outputStream.close();
    assertEquals(uncompressedSize, compressor.getBytesRead());
    assertTrue(compressor.finished());
    // just make sure we can decompress the file
    ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    Decompressor decompressor = codec.createDecompressor();
    CompressionInputStream inputStream = codec.createInputStream(bais, decompressor);
    byte[] buffer = new byte[100];
    int n = buffer.length;
    while ((n = inputStream.read(buffer, 0, n)) != -1) {
        byteArrayOutputStream.write(buffer, 0, n);
    }
    assertArrayEquals(bytes, byteArrayOutputStream.toByteArray());
}