Examples with BZip2CompressorOutputStream org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream used on opensource projects

Example 6 with BZip2CompressorOutputStream

use of org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream in project rest.li by linkedin.

the class TestStreamingCompression method testBzip2Compressor.

@Test
public void testBzip2Compressor() throws IOException, InterruptedException, CompressionException, ExecutionException {
    StreamingCompressor compressor = new Bzip2Compressor(_executor);
    final byte[] origin = new byte[BUF_SIZE];
    Arrays.fill(origin, (byte) 'c');
    ByteArrayOutputStream out = new ByteArrayOutputStream();
    BZip2CompressorOutputStream bzip = new BZip2CompressorOutputStream(out);
    IOUtils.write(origin, bzip);
    bzip.close();
    byte[] compressed = out.toByteArray();
    testCompress(compressor, origin, compressed);
    testDecompress(compressor, origin, compressed);
    testCompressThenDecompress(compressor, origin);
}

Example 7 with BZip2CompressorOutputStream

use of org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream in project DataX by alibaba.

the class UnstructuredStorageWriterUtil method writeToStream.

public static void writeToStream(RecordReceiver lineReceiver, OutputStream outputStream, Configuration config, String context, TaskPluginCollector taskPluginCollector) {
    String encoding = config.getString(Key.ENCODING, Constant.DEFAULT_ENCODING);
    // handle blank encoding
    if (StringUtils.isBlank(encoding)) {
        LOG.warn(String.format("您配置的encoding为[%s], 使用默认值[%s]", encoding, Constant.DEFAULT_ENCODING));
        encoding = Constant.DEFAULT_ENCODING;
    }
    String compress = config.getString(Key.COMPRESS);
    BufferedWriter writer = null;
    // compress logic
    try {
        if (null == compress) {
            writer = new BufferedWriter(new OutputStreamWriter(outputStream, encoding));
        } else {
            // TODO more compress
            if ("gzip".equalsIgnoreCase(compress)) {
                CompressorOutputStream compressorOutputStream = new GzipCompressorOutputStream(outputStream);
                writer = new BufferedWriter(new OutputStreamWriter(compressorOutputStream, encoding));
            } else if ("bzip2".equalsIgnoreCase(compress)) {
                CompressorOutputStream compressorOutputStream = new BZip2CompressorOutputStream(outputStream);
                writer = new BufferedWriter(new OutputStreamWriter(compressorOutputStream, encoding));
            } else {
                throw DataXException.asDataXException(UnstructuredStorageWriterErrorCode.ILLEGAL_VALUE, String.format("仅支持 gzip, bzip2 文件压缩格式 , 不支持您配置的文件压缩格式: [%s]", compress));
            }
        }
        UnstructuredStorageWriterUtil.doWriteToStream(lineReceiver, writer, context, config, taskPluginCollector);
    } catch (UnsupportedEncodingException uee) {
        throw DataXException.asDataXException(UnstructuredStorageWriterErrorCode.Write_FILE_WITH_CHARSET_ERROR, String.format("不支持的编码格式 : [%s]", encoding), uee);
    } catch (NullPointerException e) {
        throw DataXException.asDataXException(UnstructuredStorageWriterErrorCode.RUNTIME_EXCEPTION, "运行时错误, 请联系我们", e);
    } catch (IOException e) {
        throw DataXException.asDataXException(UnstructuredStorageWriterErrorCode.Write_FILE_IO_ERROR, String.format("流写入错误 : [%s]", context), e);
    } finally {
        IOUtils.closeQuietly(writer);
    }
}

Example 8 with BZip2CompressorOutputStream

use of org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream in project beam by apache.

the class TextIOTest method writeToFile.

private static File writeToFile(String[] lines, String filename, CompressionType compression) throws IOException {
    File file = tempFolder.resolve(filename).toFile();
    OutputStream output = new FileOutputStream(file);
    switch(compression) {
        case UNCOMPRESSED:
            break;
        case GZIP:
            output = new GZIPOutputStream(output);
            break;
        case BZIP2:
            output = new BZip2CompressorOutputStream(output);
            break;
        case ZIP:
            ZipOutputStream zipOutput = new ZipOutputStream(output);
            zipOutput.putNextEntry(new ZipEntry("entry"));
            output = zipOutput;
            break;
        case DEFLATE:
            output = new DeflateCompressorOutputStream(output);
            break;
        default:
            throw new UnsupportedOperationException(compression.toString());
    }
    writeToStreamAndClose(lines, output);
    return file;
}

Example 9 with BZip2CompressorOutputStream

use of org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream in project stanbol by apache.

the class IndexerImpl method getEntityIdFileOutputStream.

/**
     * Opens a stream to read data from the {@link #indexedEntityIdFile}. 
     * Can only be called in {@link State}s earlier that {@link State#INDEXING}.
     * @return the stream
     * @throws IOException on any error while opening the stream
     * @throws IllegalStateException if {@link #getState()} is later than
     * {@link State#INITIALISED}
     */
protected OutputStream getEntityIdFileOutputStream() throws IOException {
    if (indexedEntityIdFile == null) {
        return null;
    }
    State state = getState();
    if (state.ordinal() > State.INITIALISED.ordinal()) {
        throw new IllegalStateException("Opening an OutputStrem to the " + "indexed entity id file '" + indexedEntityIdFile + "' is not " + "allowed for states > " + State.INITIALISED + " (current: " + state + ")!");
    }
    if (indexedEntityIdFile.isFile()) {
        //exists
        log.info(" ... delete existing IndexedEntityId file " + indexedEntityIdFile);
        //delete existing data
        indexedEntityIdFile.delete();
    }
    //support compression
    String extension = FilenameUtils.getExtension(indexedEntityIdFile.getName());
    OutputStream out = new FileOutputStream(indexedEntityIdFile);
    if ("zip".equalsIgnoreCase(extension)) {
        out = new ZipOutputStream(out);
        ((ZipOutputStream) out).putNextEntry(new ZipEntry("entity-ids.txt"));
    } else if ("gz".equalsIgnoreCase(extension)) {
        out = new GZIPOutputStream(out);
    } else if ("bz2".equalsIgnoreCase(extension)) {
        out = new BZip2CompressorOutputStream(out);
    }
    return out;
}

Example 10 with BZip2CompressorOutputStream

use of org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream in project stanbol by apache.

the class Urify method urify.

private void urify(String resource) throws IOException {
    File source = new File(resource);
    if (source.isFile()) {
        String path = FilenameUtils.getFullPathNoEndSeparator(resource);
        String name = FilenameUtils.getName(resource);
        File target = new File(path, outputFilePrefix + name);
        int i = 0;
        while (target.exists()) {
            i++;
            target = new File(path, "uf" + i + "_" + name);
        }
        InputStream is = new FileInputStream(source);
        OutputStream os = new FileOutputStream(target);
        log.info("RDFTerm: {}", resource);
        log.info("Target  : {}", target);
        if ("gz".equalsIgnoreCase(FilenameUtils.getExtension(name))) {
            is = new GZIPInputStream(is);
            os = new GZIPOutputStream(os);
            name = FilenameUtils.removeExtension(name);
            log.debug("   - from GZIP Archive");
        } else if ("bz2".equalsIgnoreCase(FilenameUtils.getExtension(name))) {
            is = new BZip2CompressorInputStream(is);
            os = new BZip2CompressorOutputStream(os);
            name = FilenameUtils.removeExtension(name);
            log.debug("   - from BZip2 Archive");
        }
        // TODO: No Zip File support
        //else no complression
        BlockingQueue<String> queue = new ArrayBlockingQueue<String>(1000);
        ReaderDaemon reader = new ReaderDaemon(new BufferedReader(new InputStreamReader(is, charset)), queue);
        WriterDaemon writer = new WriterDaemon(new BufferedWriter(new OutputStreamWriter(os, charset)), queue);
        Thread readerDaemon = new Thread(reader, name + " reader");
        Thread writerDaemon = new Thread(writer, name + " writer");
        readerDaemon.setDaemon(true);
        writerDaemon.setDaemon(true);
        writerDaemon.start();
        readerDaemon.start();
        Object notifier = writer.getNotifier();
        synchronized (notifier) {
            //wait until processed
            if (!writer.completed()) {
                try {
                    notifier.wait();
                } catch (InterruptedException e) {
                /*ignore*/
                }
            }
        }
        if (reader.getError() != null) {
            throw new IOException("Error while reading source " + source, reader.getError());
        }
        if (writer.getError() != null) {
            throw new IOException("Error while writing resource " + target, writer.getError());
        }
        log.info(" ... completed resource {}", resource);
    } else {
        throw new FileNotFoundException("Parsed File " + resource + " does not exist or is not a File!");
    }
}