Examples with LZ4FrameInputStream net.jpountz.lz4.LZ4FrameInputStream used on opensource projects

Example 1 with LZ4FrameInputStream

use of net.jpountz.lz4.LZ4FrameInputStream in project nrtsearch by Yelp.

the class LZ4CodecTest method testDecompressWrapper.

@Test
public void testDecompressWrapper() throws IOException {
    InputStream inputStream = new ByteArrayInputStream(new byte[0]);
    InputStream wrappedStream = LZ4Codec.INSTANCE.decompress(inputStream);
    assertTrue(wrappedStream instanceof LZ4FrameInputStream);
}

Example 2 with LZ4FrameInputStream

use of net.jpountz.lz4.LZ4FrameInputStream in project nrtsearch by Yelp.

the class TarImpl method extractTar.

@Override
public void extractTar(Path sourceFile, Path destDir) throws IOException {
    final FileInputStream fileInputStream = new FileInputStream(sourceFile.toFile());
    final InputStream compressorInputStream;
    if (compressionMode.equals(CompressionMode.LZ4)) {
        compressorInputStream = new LZ4FrameInputStream(fileInputStream);
    } else {
        compressorInputStream = new GzipCompressorInputStream(fileInputStream, true);
    }
    try (final TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(compressorInputStream)) {
        extractTar(tarArchiveInputStream, destDir);
    }
}

Example 3 with LZ4FrameInputStream

use of net.jpountz.lz4.LZ4FrameInputStream in project nrtsearch by Yelp.

the class ArchiverTest method testUploadWithParameters.

private void testUploadWithParameters(String service, String resource, Path sourceDir, List<String> includeFiles, List<String> includeDirs, List<String> ignoreVerifying, boolean stream) throws IOException {
    String versionHash = archiver.upload(service, resource, sourceDir, includeFiles, includeDirs, stream);
    Path actualDownloadDir = Files.createDirectory(archiverDirectory.resolve("actualDownload"));
    try (S3Object s3Object = s3.getObject(BUCKET_NAME, String.format("%s/%s/%s", service, resource, versionHash));
        S3ObjectInputStream s3ObjectInputStream = s3Object.getObjectContent();
        LZ4FrameInputStream lz4CompressorInputStream = new LZ4FrameInputStream(s3ObjectInputStream);
        TarArchiveInputStream tarArchiveInputStream = new TarArchiveInputStream(lz4CompressorInputStream)) {
        new TarImpl(TarImpl.CompressionMode.LZ4).extractTar(tarArchiveInputStream, actualDownloadDir);
    }
    assertTrue(TarImplTest.dirsMatch(actualDownloadDir.resolve(resource).toFile(), sourceDir.toFile(), ignoreVerifying));
    rmDir(actualDownloadDir);
}

Example 4 with LZ4FrameInputStream

use of net.jpountz.lz4.LZ4FrameInputStream in project batfish by batfish.

the class FileBasedStorage method deserializeObject.

/**
 * Returns a single object of the given class deserialized from the given file. Uses the {@link
 * FileBasedStorage} default file encoding including serialization format and compression.
 */
// PMD does not understand Closer
@SuppressWarnings("PMD.CloseResource")
private <S extends Serializable> S deserializeObject(Path inputFile, Class<S> outputClass) throws BatfishException {
    Path sanitizedInputFile = validatePath(inputFile);
    try (Closer closer = Closer.create()) {
        FileInputStream fis = closer.register(new FileInputStream(sanitizedInputFile.toFile()));
        PushbackInputStream pbstream = new PushbackInputStream(fis, DEFAULT_HEADER_LENGTH_BYTES);
        Format f = detectFormat(pbstream);
        ObjectInputStream ois;
        if (f == Format.GZIP) {
            GZIPInputStream gis = closer.register(new GZIPInputStream(pbstream, 8192));
            ois = new ObjectInputStream(gis);
        } else if (f == Format.LZ4) {
            LZ4FrameInputStream lis = closer.register(new LZ4FrameInputStream(pbstream));
            ois = new ObjectInputStream(lis);
        } else if (f == Format.JAVA_SERIALIZED) {
            ois = new ObjectInputStream(pbstream);
        } else {
            throw new BatfishException(String.format("Could not detect format of the file %s", sanitizedInputFile));
        }
        closer.register(ois);
        return outputClass.cast(ois.readObject());
    } catch (Exception e) {
        throw new BatfishException(String.format("Failed to deserialize object of type %s from file %s", outputClass.getCanonicalName(), sanitizedInputFile), e);
    }
}

Example 5 with LZ4FrameInputStream

use of net.jpountz.lz4.LZ4FrameInputStream in project batfish by batfish.

the class FileBasedStorage method deserializeObjectUnchecked.

// PMD does not understand Closer
@SuppressWarnings("PMD.CloseResource")
private <S extends Serializable> S deserializeObjectUnchecked(Path inputFile) throws BatfishException {
    Path sanitizedInputFile = validatePath(inputFile);
    try (Closer closer = Closer.create()) {
        FileInputStream fis = closer.register(new FileInputStream(sanitizedInputFile.toFile()));
        PushbackInputStream pbstream = closer.register(new PushbackInputStream(fis, DEFAULT_HEADER_LENGTH_BYTES));
        Format f = detectFormat(pbstream);
        InputStream ois;
        if (f == Format.GZIP) {
            ois = closer.register(new GZIPInputStream(pbstream, 8192));
        } else if (f == Format.LZ4) {
            ois = closer.register(new LZ4FrameInputStream(pbstream));
        } else if (f == Format.JAVA_SERIALIZED) {
            ois = pbstream;
        } else {
            throw new BatfishException(String.format("Could not detect format of the file %s", sanitizedInputFile));
        }
        return SerializationUtils.deserialize(ois);
    } catch (Exception e) {
        throw new BatfishException(String.format("Failed to deserialize object from file %s", sanitizedInputFile), e);
    }
}