Examples with ZlibCompressor org.apache.hadoop.io.compress.zlib.ZlibCompressor used on opensource projects

Example 1 with ZlibCompressor

use of org.apache.hadoop.io.compress.zlib.ZlibCompressor in project hadoop by apache.

the class TestCodec method testGzipCodecWrite.

private void testGzipCodecWrite(boolean useNative) throws IOException {
    // Create a gzipped file using a compressor from the CodecPool,
    // and try to read it back via the regular GZIPInputStream.
    // Use native libs per the parameter
    Configuration conf = new Configuration();
    if (useNative) {
        assumeTrue(ZlibFactory.isNativeZlibLoaded(conf));
    } else {
        assertFalse("ZlibFactory is using native libs against request", ZlibFactory.isNativeZlibLoaded(conf));
    }
    // Ensure that the CodecPool has a BuiltInZlibDeflater in it.
    Compressor zlibCompressor = ZlibFactory.getZlibCompressor(conf);
    assertNotNull("zlibCompressor is null!", zlibCompressor);
    assertTrue("ZlibFactory returned unexpected deflator", useNative ? zlibCompressor instanceof ZlibCompressor : zlibCompressor instanceof BuiltInZlibDeflater);
    CodecPool.returnCompressor(zlibCompressor);
    // Create a GZIP text file via the Compressor interface.
    CompressionCodecFactory ccf = new CompressionCodecFactory(conf);
    CompressionCodec codec = ccf.getCodec(new Path("foo.gz"));
    assertTrue("Codec for .gz file is not GzipCodec", codec instanceof GzipCodec);
    final String msg = "This is the message we are going to compress.";
    final String fileName = new Path(GenericTestUtils.getTempPath("testGzipCodecWrite.txt.gz")).toString();
    BufferedWriter w = null;
    Compressor gzipCompressor = CodecPool.getCompressor(codec);
    if (null != gzipCompressor) {
        // If it gives us back a Compressor, we should be able to use this
        // to write files we can then read back with Java's gzip tools.
        OutputStream os = new CompressorStream(new FileOutputStream(fileName), gzipCompressor);
        w = new BufferedWriter(new OutputStreamWriter(os));
        w.write(msg);
        w.close();
        CodecPool.returnCompressor(gzipCompressor);
        verifyGzipFile(fileName, msg);
    }
    // Create a gzip text file via codec.getOutputStream().
    w = new BufferedWriter(new OutputStreamWriter(codec.createOutputStream(new FileOutputStream(fileName))));
    w.write(msg);
    w.close();
    verifyGzipFile(fileName, msg);
}

Example 2 with ZlibCompressor

use of org.apache.hadoop.io.compress.zlib.ZlibCompressor in project hadoop by apache.

the class TestCodec method testCodecPoolGzipReuse.

@Test
public void testCodecPoolGzipReuse() throws Exception {
    Configuration conf = new Configuration();
    assumeTrue(ZlibFactory.isNativeZlibLoaded(conf));
    GzipCodec gzc = ReflectionUtils.newInstance(GzipCodec.class, conf);
    DefaultCodec dfc = ReflectionUtils.newInstance(DefaultCodec.class, conf);
    Compressor c1 = CodecPool.getCompressor(gzc);
    Compressor c2 = CodecPool.getCompressor(dfc);
    CodecPool.returnCompressor(c1);
    CodecPool.returnCompressor(c2);
    assertTrue("Got mismatched ZlibCompressor", c2 != CodecPool.getCompressor(gzc));
}

Example 3 with ZlibCompressor

use of org.apache.hadoop.io.compress.zlib.ZlibCompressor in project hadoop by apache.

the class TestCodec method gzipReinitTest.

private static void gzipReinitTest(Configuration conf, CompressionCodec codec) throws IOException {
    // Add codec to cache
    ZlibFactory.setCompressionLevel(conf, CompressionLevel.BEST_COMPRESSION);
    ZlibFactory.setCompressionStrategy(conf, CompressionStrategy.DEFAULT_STRATEGY);
    Compressor c1 = CodecPool.getCompressor(codec);
    CodecPool.returnCompressor(c1);
    // reset compressor's compression level to perform no compression
    ZlibFactory.setCompressionLevel(conf, CompressionLevel.NO_COMPRESSION);
    Compressor c2 = CodecPool.getCompressor(codec, conf);
    // ensure same compressor placed earlier
    assertTrue("Got mismatched ZlibCompressor", c1 == c2);
    ByteArrayOutputStream bos = new ByteArrayOutputStream();
    CompressionOutputStream cos = null;
    // write trivially compressable data
    byte[] b = new byte[1 << 15];
    Arrays.fill(b, (byte) 43);
    try {
        cos = codec.createOutputStream(bos, c2);
        cos.write(b);
    } finally {
        if (cos != null) {
            cos.close();
        }
        CodecPool.returnCompressor(c2);
    }
    byte[] outbytes = bos.toByteArray();
    // verify data were not compressed
    assertTrue("Compressed bytes contrary to configuration", outbytes.length >= b.length);
}