Examples with CompressionCodec org.apache.hadoop.io.compress.CompressionCodec used on opensource projects

Example 76 with CompressionCodec

use of org.apache.hadoop.io.compress.CompressionCodec in project hadoop by apache.

the class CompressionEmulationUtil method configureCompressionEmulation.

/**
   * Extracts compression/decompression related configuration parameters from 
   * the source configuration to the target configuration.
   */
static void configureCompressionEmulation(Configuration source, Configuration target) {
    // enable output compression
    target.setBoolean(FileOutputFormat.COMPRESS, source.getBoolean(FileOutputFormat.COMPRESS, false));
    // set the job output compression codec
    String jobOutputCompressionCodec = source.get(FileOutputFormat.COMPRESS_CODEC);
    if (jobOutputCompressionCodec != null) {
        target.set(FileOutputFormat.COMPRESS_CODEC, jobOutputCompressionCodec);
    }
    // set the job output compression type
    String jobOutputCompressionType = source.get(FileOutputFormat.COMPRESS_TYPE);
    if (jobOutputCompressionType != null) {
        target.set(FileOutputFormat.COMPRESS_TYPE, jobOutputCompressionType);
    }
    // enable map output compression
    target.setBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, source.getBoolean(MRJobConfig.MAP_OUTPUT_COMPRESS, false));
    // set the map output compression codecs
    String mapOutputCompressionCodec = source.get(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC);
    if (mapOutputCompressionCodec != null) {
        target.set(MRJobConfig.MAP_OUTPUT_COMPRESS_CODEC, mapOutputCompressionCodec);
    }
    // enable input decompression
    //TODO replace with mapInputBytes and hdfsBytesRead
    Path[] inputs = org.apache.hadoop.mapred.FileInputFormat.getInputPaths(new JobConf(source));
    boolean needsCompressedInput = false;
    CompressionCodecFactory compressionCodecs = new CompressionCodecFactory(source);
    for (Path input : inputs) {
        CompressionCodec codec = compressionCodecs.getCodec(input);
        if (codec != null) {
            needsCompressedInput = true;
        }
    }
    setInputCompressionEmulationEnabled(target, needsCompressedInput);
}

Example 77 with CompressionCodec

use of org.apache.hadoop.io.compress.CompressionCodec in project hadoop by apache.

the class Anonymizer method createJsonGenerator.

// Creates a JSON generator
private JsonGenerator createJsonGenerator(Configuration conf, Path path) throws IOException {
    FileSystem outFS = path.getFileSystem(conf);
    CompressionCodec codec = new CompressionCodecFactory(conf).getCodec(path);
    OutputStream output;
    Compressor compressor = null;
    if (codec != null) {
        compressor = CodecPool.getCompressor(codec);
        output = codec.createOutputStream(outFS.create(path), compressor);
    } else {
        output = outFS.create(path);
    }
    JsonGenerator outGen = outFactory.createGenerator(output, JsonEncoding.UTF8);
    outGen.useDefaultPrettyPrinter();
    return outGen;
}

Example 78 with CompressionCodec

use of org.apache.hadoop.io.compress.CompressionCodec in project hadoop by apache.

the class FSImageUtil method wrapInputStreamForCompression.

public static InputStream wrapInputStreamForCompression(Configuration conf, String codec, InputStream in) throws IOException {
    if (codec.isEmpty())
        return in;
    FSImageCompression compression = FSImageCompression.createCompression(conf, codec);
    CompressionCodec imageCodec = compression.getImageCodec();
    return imageCodec.createInputStream(in);
}

Example 79 with CompressionCodec

use of org.apache.hadoop.io.compress.CompressionCodec in project carbondata by apache.

the class FileFactory method getDataInputStream.

public static DataInputStream getDataInputStream(String path, FileType fileType, int bufferSize) throws IOException {
    path = path.replace("\\", "/");
    boolean gzip = path.endsWith(".gz");
    boolean bzip2 = path.endsWith(".bz2");
    InputStream stream;
    switch(fileType) {
        case LOCAL:
            path = getUpdatedFilePath(path, fileType);
            if (gzip) {
                stream = new GZIPInputStream(new FileInputStream(path));
            } else if (bzip2) {
                stream = new BZip2CompressorInputStream(new FileInputStream(path));
            } else {
                stream = new FileInputStream(path);
            }
            break;
        case HDFS:
        case ALLUXIO:
        case VIEWFS:
            Path pt = new Path(path);
            FileSystem fs = pt.getFileSystem(configuration);
            if (bufferSize == -1) {
                stream = fs.open(pt);
            } else {
                stream = fs.open(pt, bufferSize);
            }
            String codecName = null;
            if (gzip) {
                codecName = GzipCodec.class.getName();
            } else if (bzip2) {
                codecName = BZip2Codec.class.getName();
            }
            if (null != codecName) {
                CompressionCodecFactory ccf = new CompressionCodecFactory(configuration);
                CompressionCodec codec = ccf.getCodecByClassName(codecName);
                stream = codec.createInputStream(stream);
            }
            break;
        default:
            throw new UnsupportedOperationException("unsupported file system");
    }
    return new DataInputStream(new BufferedInputStream(stream));
}

Example 80 with CompressionCodec

use of org.apache.hadoop.io.compress.CompressionCodec in project jena by apache.

the class AbstractCompressedNodeTupleInputFormatTests method getOutputStream.

@Override
protected OutputStream getOutputStream(File f) throws IOException {
    CompressionCodec codec = this.getCompressionCodec();
    if (codec instanceof Configurable) {
        ((Configurable) codec).setConf(this.prepareConfiguration());
    }
    FileOutputStream fileOutput = new FileOutputStream(f, false);
    return codec.createOutputStream(fileOutput);
}