Examples with SnappyInputStream org.xerial.snappy.SnappyInputStream used on opensource projects

Example 1 with SnappyInputStream

use of org.xerial.snappy.SnappyInputStream in project nimbus by nimbus-org.

the class SerializableExternalizerService method readExternal.

public Object readExternal(ObjectInput in) throws IOException, ClassNotFoundException {
    if (compressMode == COMPRESS_MODE_NONE) {
        return readInternal(in);
    }
    final boolean isCompressed = in.readBoolean();
    final int length = in.readInt();
    if (length == 0) {
        return null;
    }
    final byte[] bytes = new byte[length];
    final ByteArrayOutputStream baos = new ByteArrayOutputStream();
    int readLen = 0;
    int offset = 0;
    while ((readLen = in.read(bytes, offset, length - offset)) != -1) {
        baos.write(bytes, offset, readLen);
        offset += readLen;
        if (length - offset == 0) {
            break;
        }
    }
    final ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    InputStream is = null;
    Inflater inflater = null;
    if (isCompressed) {
        switch(compressMode) {
            case COMPRESS_MODE_ZLIB:
                inflater = new Inflater();
                is = bufferSize > 0 ? new InflaterInputStream(bais, inflater, bufferSize) : new InflaterInputStream(bais);
                break;
            case COMPRESS_MODE_ZIP:
                is = new ZipInputStream(bais);
                ((ZipInputStream) is).getNextEntry();
                break;
            case COMPRESS_MODE_GZIP:
                is = bufferSize > 0 ? new GZIPInputStream(bais, bufferSize) : new GZIPInputStream(bais);
                break;
            case COMPRESS_MODE_SNAPPY:
                is = new SnappyInputStream(bais);
                break;
            case COMPRESS_MODE_LZ4:
                is = new LZ4BlockInputStream(bais);
                break;
            default:
                throw new IOException("Unknown compress mode : " + compressMode);
        }
    } else {
        is = bais;
    }
    final ObjectInput oi = createObjectInput(is);
    try {
        return readInternal(oi);
    } finally {
        if (inflater != null) {
            inflater.end();
        }
        try {
            is.close();
        } catch (IOException e) {
        }
        bais.close();
    }
}

Example 2 with SnappyInputStream

use of org.xerial.snappy.SnappyInputStream in project nimbus by nimbus-org.

the class HttpResponseImpl method decompress.

/**
 * 入力ストリームの圧縮を解除する。<p>
 * (Content-Encodingに指定された逆順で解除)
 *
 * @param is 入力ストリーム
 * @return 圧縮解除された入力ストリーム
 * @throws IOException サポートしていない圧縮形式(deflate, gzip以外)が指定された場合
 */
protected InputStream decompress(InputStream is) throws IOException {
    if (is == null) {
        return null;
    }
    // ヘッダー[Content-Encoding]の値を取得
    String encode = getHeader(HEADER_CONTENT_ENCODING);
    InputStream in = is;
    if (encode != null) {
        if (encode.indexOf(CONTENT_ENCODING_DEFLATE) != -1) {
            // deflate圧縮解除
            in = new InflaterInputStream(in);
        } else if (encode.indexOf(CONTENT_ENCODING_GZIP) != -1 || encode.indexOf(CONTENT_ENCODING_X_GZIP) != -1) {
            // gzip圧縮解除
            in = new GZIPInputStream(in);
        } else if (encode.indexOf(CONTENT_ENCODING_SNAPPY) != -1) {
            in = new SnappyInputStream(in);
        } else if (encode.indexOf(CONTENT_ENCODING_LZ4) != -1) {
            in = new LZ4BlockInputStream(in);
        } else {
            throw new IOException("Can not decompress. [" + encode + "]");
        }
    }
    String transferEncoding = getHeader(HEADER_TRANSFER_ENCODING);
    if (isConnectionClose() && (getContentLength() > 0 || (transferEncoding != null && transferEncoding.indexOf(TRANSFER_ENCODING_CHUNKED) != -1))) {
        final ByteArrayOutputStream baos = new ByteArrayOutputStream();
        final byte[] bytes = new byte[1024];
        int length = 0;
        while ((length = in.read(bytes)) != -1) {
            baos.write(bytes, 0, length);
        }
        outputBytes = baos.toByteArray();
        final ByteArrayInputStream bais = new ByteArrayInputStream(outputBytes);
        return bais;
    } else {
        return in;
    }
}

Example 3 with SnappyInputStream

use of org.xerial.snappy.SnappyInputStream in project voltdb by VoltDB.

the class OnDemandBinaryLogger method main.

public static void main(String[] args) throws Exception {
    final String op = args[0];
    final String file = args[1];
    int tail = 0;
    if (op.equalsIgnoreCase("tail")) {
        tail = Integer.valueOf(args[2]);
    }
    long txnid = 0;
    if (op.equalsIgnoreCase("grep")) {
        txnid = Long.valueOf(args[2]);
    }
    FileInputStream fis = new FileInputStream(file);
    fis.getChannel().position(8);
    final long count = fis.getChannel().map(MapMode.READ_ONLY, 0, 8).order(ByteOrder.nativeOrder()).getLong();
    SnappyInputStream sis = new SnappyInputStream(fis);
    DataInputStream dis = new DataInputStream(sis);
    if (op.equalsIgnoreCase("tail")) {
        long skip = count > tail ? count - tail : 0;
        for (long ii = 0; ii < skip; ii++) {
            dis.readLong();
        }
        for (int ii = 0; ii < tail; ii++) {
            System.out.println(dis.readLong());
        }
    } else if (op.equalsIgnoreCase("grep")) {
        for (long ii = 0; ii < count; ii++) {
            if (dis.readLong() == txnid) {
                System.out.println(ii + ": " + txnid);
            }
        }
    } else {
        System.err.println("Unsupported operation " + op);
    }
}

Example 4 with SnappyInputStream

use of org.xerial.snappy.SnappyInputStream in project nifi by apache.

the class CompressContent method onTrigger.

@Override
public void onTrigger(final ProcessContext context, final ProcessSession session) {
    FlowFile flowFile = session.get();
    if (flowFile == null) {
        return;
    }
    final ComponentLog logger = getLogger();
    final long sizeBeforeCompression = flowFile.getSize();
    final String compressionMode = context.getProperty(MODE).getValue();
    String compressionFormatValue = context.getProperty(COMPRESSION_FORMAT).getValue();
    if (compressionFormatValue.equals(COMPRESSION_FORMAT_ATTRIBUTE)) {
        final String mimeType = flowFile.getAttribute(CoreAttributes.MIME_TYPE.key());
        if (mimeType == null) {
            logger.error("No {} attribute exists for {}; routing to failure", new Object[] { CoreAttributes.MIME_TYPE.key(), flowFile });
            session.transfer(flowFile, REL_FAILURE);
            return;
        }
        compressionFormatValue = compressionFormatMimeTypeMap.get(mimeType);
        if (compressionFormatValue == null) {
            logger.info("Mime Type of {} is '{}', which does not indicate a supported Compression Format; routing to success without decompressing", new Object[] { flowFile, mimeType });
            session.transfer(flowFile, REL_SUCCESS);
            return;
        }
    }
    final String compressionFormat = compressionFormatValue;
    final AtomicReference<String> mimeTypeRef = new AtomicReference<>(null);
    final StopWatch stopWatch = new StopWatch(true);
    final String fileExtension;
    switch(compressionFormat.toLowerCase()) {
        case COMPRESSION_FORMAT_GZIP:
            fileExtension = ".gz";
            break;
        case COMPRESSION_FORMAT_LZMA:
            fileExtension = ".lzma";
            break;
        case COMPRESSION_FORMAT_XZ_LZMA2:
            fileExtension = ".xz";
            break;
        case COMPRESSION_FORMAT_BZIP2:
            fileExtension = ".bz2";
            break;
        case COMPRESSION_FORMAT_SNAPPY:
            fileExtension = ".snappy";
            break;
        case COMPRESSION_FORMAT_SNAPPY_FRAMED:
            fileExtension = ".sz";
            break;
        default:
            fileExtension = "";
            break;
    }
    try {
        flowFile = session.write(flowFile, new StreamCallback() {

            @Override
            public void process(final InputStream rawIn, final OutputStream rawOut) throws IOException {
                final OutputStream compressionOut;
                final InputStream compressionIn;
                final OutputStream bufferedOut = new BufferedOutputStream(rawOut, 65536);
                final InputStream bufferedIn = new BufferedInputStream(rawIn, 65536);
                try {
                    if (MODE_COMPRESS.equalsIgnoreCase(compressionMode)) {
                        compressionIn = bufferedIn;
                        switch(compressionFormat.toLowerCase()) {
                            case COMPRESSION_FORMAT_GZIP:
                                final int compressionLevel = context.getProperty(COMPRESSION_LEVEL).asInteger();
                                compressionOut = new GZIPOutputStream(bufferedOut, compressionLevel);
                                mimeTypeRef.set("application/gzip");
                                break;
                            case COMPRESSION_FORMAT_LZMA:
                                compressionOut = new LzmaOutputStream.Builder(bufferedOut).build();
                                mimeTypeRef.set("application/x-lzma");
                                break;
                            case COMPRESSION_FORMAT_XZ_LZMA2:
                                compressionOut = new XZOutputStream(bufferedOut, new LZMA2Options());
                                mimeTypeRef.set("application/x-xz");
                                break;
                            case COMPRESSION_FORMAT_SNAPPY:
                                compressionOut = new SnappyOutputStream(bufferedOut);
                                mimeTypeRef.set("application/x-snappy");
                                break;
                            case COMPRESSION_FORMAT_SNAPPY_FRAMED:
                                compressionOut = new SnappyFramedOutputStream(bufferedOut);
                                mimeTypeRef.set("application/x-snappy-framed");
                                break;
                            case COMPRESSION_FORMAT_BZIP2:
                            default:
                                mimeTypeRef.set("application/x-bzip2");
                                compressionOut = new CompressorStreamFactory().createCompressorOutputStream(compressionFormat.toLowerCase(), bufferedOut);
                                break;
                        }
                    } else {
                        compressionOut = bufferedOut;
                        switch(compressionFormat.toLowerCase()) {
                            case COMPRESSION_FORMAT_LZMA:
                                compressionIn = new LzmaInputStream(bufferedIn, new Decoder());
                                break;
                            case COMPRESSION_FORMAT_XZ_LZMA2:
                                compressionIn = new XZInputStream(bufferedIn);
                                break;
                            case COMPRESSION_FORMAT_BZIP2:
                                // need this two-arg constructor to support concatenated streams
                                compressionIn = new BZip2CompressorInputStream(bufferedIn, true);
                                break;
                            case COMPRESSION_FORMAT_GZIP:
                                compressionIn = new GzipCompressorInputStream(bufferedIn, true);
                                break;
                            case COMPRESSION_FORMAT_SNAPPY:
                                compressionIn = new SnappyInputStream(bufferedIn);
                                break;
                            case COMPRESSION_FORMAT_SNAPPY_FRAMED:
                                compressionIn = new SnappyFramedInputStream(bufferedIn);
                                break;
                            default:
                                compressionIn = new CompressorStreamFactory().createCompressorInputStream(compressionFormat.toLowerCase(), bufferedIn);
                        }
                    }
                } catch (final Exception e) {
                    closeQuietly(bufferedOut);
                    throw new IOException(e);
                }
                try (final InputStream in = compressionIn;
                    final OutputStream out = compressionOut) {
                    final byte[] buffer = new byte[8192];
                    int len;
                    while ((len = in.read(buffer)) > 0) {
                        out.write(buffer, 0, len);
                    }
                    out.flush();
                }
            }
        });
        stopWatch.stop();
        final long sizeAfterCompression = flowFile.getSize();
        if (MODE_DECOMPRESS.equalsIgnoreCase(compressionMode)) {
            flowFile = session.removeAttribute(flowFile, CoreAttributes.MIME_TYPE.key());
            if (context.getProperty(UPDATE_FILENAME).asBoolean()) {
                final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
                if (filename.toLowerCase().endsWith(fileExtension)) {
                    flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), filename.substring(0, filename.length() - fileExtension.length()));
                }
            }
        } else {
            flowFile = session.putAttribute(flowFile, CoreAttributes.MIME_TYPE.key(), mimeTypeRef.get());
            if (context.getProperty(UPDATE_FILENAME).asBoolean()) {
                final String filename = flowFile.getAttribute(CoreAttributes.FILENAME.key());
                flowFile = session.putAttribute(flowFile, CoreAttributes.FILENAME.key(), filename + fileExtension);
            }
        }
        logger.info("Successfully {}ed {} using {} compression format; size changed from {} to {} bytes", new Object[] { compressionMode.toLowerCase(), flowFile, compressionFormat, sizeBeforeCompression, sizeAfterCompression });
        session.getProvenanceReporter().modifyContent(flowFile, stopWatch.getDuration(TimeUnit.MILLISECONDS));
        session.transfer(flowFile, REL_SUCCESS);
    } catch (final ProcessException e) {
        logger.error("Unable to {} {} using {} compression format due to {}; routing to failure", new Object[] { compressionMode.toLowerCase(), flowFile, compressionFormat, e });
        session.transfer(flowFile, REL_FAILURE);
    }
}

Example 5 with SnappyInputStream

use of org.xerial.snappy.SnappyInputStream in project carbondata by apache.

the class LocalCarbonFile method getDataInputStream.

@Override
public DataInputStream getDataInputStream(String path, FileFactory.FileType fileType, int bufferSize, String compressor) throws IOException {
    path = path.replace("\\", "/");
    path = FileFactory.getUpdatedFilePath(path, fileType);
    InputStream inputStream;
    if (compressor.isEmpty()) {
        inputStream = new FileInputStream(path);
    } else if ("GZIP".equalsIgnoreCase(compressor)) {
        inputStream = new GZIPInputStream(new FileInputStream(path));
    } else if ("BZIP2".equalsIgnoreCase(compressor)) {
        inputStream = new BZip2CompressorInputStream(new FileInputStream(path));
    } else if ("SNAPPY".equalsIgnoreCase(compressor)) {
        inputStream = new SnappyInputStream(new FileInputStream(path));
    } else if ("LZ4".equalsIgnoreCase(compressor)) {
        inputStream = new LZ4BlockInputStream(new FileInputStream(path));
    } else {
        throw new IOException("Unsupported compressor: " + compressor);
    }
    if (bufferSize <= 0) {
        return new DataInputStream(new BufferedInputStream(inputStream));
    } else {
        return new DataInputStream(new BufferedInputStream(inputStream, bufferSize));
    }
}