Examples with BuiltInZlibDeflater org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater used on opensource projects

Example 1 with BuiltInZlibDeflater

use of org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater in project hadoop by apache.

the class TestCodec method testGzipCodecWrite.

private void testGzipCodecWrite(boolean useNative) throws IOException {
    // Create a gzipped file using a compressor from the CodecPool,
    // and try to read it back via the regular GZIPInputStream.
    // Use native libs per the parameter
    Configuration conf = new Configuration();
    if (useNative) {
        assumeTrue(ZlibFactory.isNativeZlibLoaded(conf));
    } else {
        assertFalse("ZlibFactory is using native libs against request", ZlibFactory.isNativeZlibLoaded(conf));
    }
    // Ensure that the CodecPool has a BuiltInZlibDeflater in it.
    Compressor zlibCompressor = ZlibFactory.getZlibCompressor(conf);
    assertNotNull("zlibCompressor is null!", zlibCompressor);
    assertTrue("ZlibFactory returned unexpected deflator", useNative ? zlibCompressor instanceof ZlibCompressor : zlibCompressor instanceof BuiltInZlibDeflater);
    CodecPool.returnCompressor(zlibCompressor);
    // Create a GZIP text file via the Compressor interface.
    CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
    CompressionCodec codec = ccf.getCodec(new Path("foo.gz"));
    assertTrue("Codec for .gz file is not GzipCodec", codec instanceof GzipCodec);
    final String msg = "This is the message we are going to compress.";
    final String fileName = new Path(GenericTestUtils.getTempPath("testGzipCodecWrite.txt.gz")).toString();
    BufferedWriter w = null;
    Compressor gzipCompressor = CodecPool.getCompressor(codec);
    if (null != gzipCompressor) {
        // If it gives us back a Compressor, we should be able to use this
        // to write files we can then read back with Java's gzip tools.
        OutputStream os = new CompressorStream(new FileOutputStream(fileName), gzipCompressor);
        w = new BufferedWriter(new OutputStreamWriter(os));
        w.write(msg);
        w.close();
        CodecPool.returnCompressor(gzipCompressor);
        verifyGzipFile(fileName, msg);
    }
    // Create a gzip text file via codec.getOutputStream().
    w = new BufferedWriter(new OutputStreamWriter(codec.createOutputStream(new FileOutputStream(fileName))));
    w.write(msg);
    w.close();
    verifyGzipFile(fileName, msg);
}

Example 2 with BuiltInZlibDeflater

use of org.apache.hadoop.io.compress.zlib.BuiltInZlibDeflater in project hadoop by apache.

the class TestCompressorDecompressor method testCompressorDecompressor.

@Test
public void testCompressorDecompressor() {
    // no more for this data
    int SIZE = 44 * 1024;
    byte[] rawData = generate(SIZE);
    try {
        CompressDecompressTester.of(rawData).withCompressDecompressPair(new SnappyCompressor(), new SnappyDecompressor()).withCompressDecompressPair(new Lz4Compressor(), new Lz4Decompressor()).withCompressDecompressPair(new BuiltInZlibDeflater(), new BuiltInZlibInflater()).withTestCases(ImmutableSet.of(CompressionTestStrategy.COMPRESS_DECOMPRESS_SINGLE_BLOCK, CompressionTestStrategy.COMPRESS_DECOMPRESS_BLOCK, CompressionTestStrategy.COMPRESS_DECOMPRESS_ERRORS, CompressionTestStrategy.COMPRESS_DECOMPRESS_WITH_EMPTY_STREAM)).test();
    } catch (Exception ex) {
        GenericTestUtils.assertExceptionContains("testCompressorDecompressor error !!!", ex);
    }
}