Examples with ArchiveInputStream org.apache.commons.compress.archivers.ArchiveInputStream used on opensource projects

Example 21 with ArchiveInputStream

use of org.apache.commons.compress.archivers.ArchiveInputStream in project agile-service by open-hand.

the class StaticFileCompressServiceImpl method unCompressedByApache.

/**
 * 可使用apache解压工具进行解压的流程一致，根据文件后缀名获取不同的压缩流
 *
 * @param staticFileCompress            解压参数
 * @param projectId                     项目id
 * @param organizationId                组织id
 * @param suffix                        文件后缀名
 * @param staticFileCompressHistoryList 解压操作历史记录
 * @throws IOException io错误
 */
private void unCompressedByApache(StaticFileCompressDTO staticFileCompress, Long projectId, Long organizationId, String suffix, List<StaticFileOperationHistoryDTO> staticFileCompressHistoryList) throws IOException {
    Long userId = DetailsHelper.getUserDetails().getUserId();
    StaticFileHeaderDTO update = new StaticFileHeaderDTO();
    update.setId(staticFileCompress.getId());
    int size = staticFileCompress.getSize();
    double process = 0.0;
    List<StaticFileLineDTO> lineList = new ArrayList<>();
    List<String> urls = new ArrayList<>();
    String prefixPath = staticFileCompress.getPrefixPath();
    try (BufferedInputStream bufferedInputStream = new BufferedInputStream(staticFileCompress.getIn());
        ArchiveInputStream in = getArchiveInputStream(bufferedInputStream, suffix, staticFileCompress.getEncode())) {
        ArchiveEntry entry;
        while (Objects.nonNull(entry = in.getNextEntry())) {
            int availableSize = bufferedInputStream.available();
            if (!entry.isDirectory() && in.canReadEntryData(entry)) {
                byte[] bytes = inputToByte(in);
                int newSize = bytes.length;
                // 跳过文件夹与不能读取数据的项
                if (entry.getName().contains(MACOSX) || entry.getName().contains(DS_STORE) || newSize <= 0) {
                    // 跳过冗余文件
                    continue;
                }
                // 文件上传
                String url = fileClient.uploadFile(organizationId, BUCKET_NAME, null, getEntryFileName(entry.getName()), bytes);
                urls.add(url);
                StaticFileLineDTO staticFileLine = new StaticFileLineDTO(projectId, organizationId, staticFileCompress.getId(), dealUrl(url), dealRelativePath(entry.getName(), prefixPath));
                lineList.add(staticFileLine);
            }
            process = updateProcess(staticFileCompressHistoryList, staticFileCompress.getStaticFileCompressHistory(), size, (size - availableSize), process, staticFileCompress.getIssueId());
        }
        // 获取上传的文件信息
        List<FileDTO> files = fileClient.getFiles(organizationId, BUCKET_NAME, urls);
        Map<String, FileDTO> fileMap = files.stream().collect(Collectors.toMap(file -> dealUrl(file.getFileUrl()), file -> file));
        lineList.forEach(line -> {
            // 设置行的文件类型及其记录其他信息
            line.setId(snowflakeHelper.next());
            line.setCreatedBy(userId);
            line.setLastUpdatedBy(userId);
            line.setFileType(fileMap.get(line.getUrl()) != null ? fileMap.get(line.getUrl()).getFileType() : null);
        });
        staticFileLineMapper.batchInsert(lineList);
        updateHistoryStatus(staticFileCompress.getStaticFileCompressHistory(), SUCCESS);
        staticFileCompress.setStatus(SUCCESS);
        sendProcess(staticFileCompressHistoryList, staticFileCompress.getStaticFileCompressHistory().getUserId(), projectId, staticFileCompress.getIssueId());
    }
}

Example 22 with ArchiveInputStream

use of org.apache.commons.compress.archivers.ArchiveInputStream in project fess-crawler by codelibs.

the class TarExtractor method getTextInternal.

protected String getTextInternal(final InputStream in, final MimeTypeHelper mimeTypeHelper, final ExtractorFactory extractorFactory) {
    final StringBuilder buf = new StringBuilder(1000);
    ArchiveInputStream ais = null;
    try {
        ais = archiveStreamFactory.createArchiveInputStream("tar", in);
        TarArchiveEntry entry = null;
        long contentSize = 0;
        while ((entry = (TarArchiveEntry) ais.getNextEntry()) != null) {
            contentSize += entry.getSize();
            if (maxContentSize != -1 && contentSize > maxContentSize) {
                throw new MaxLengthExceededException("Extracted size is " + contentSize + " > " + maxContentSize);
            }
            final String filename = entry.getName();
            final String mimeType = mimeTypeHelper.getContentType(null, filename);
            if (mimeType != null) {
                final Extractor extractor = extractorFactory.getExtractor(mimeType);
                if (extractor != null) {
                    try {
                        final Map<String, String> map = new HashMap<>();
                        map.put(ExtractData.RESOURCE_NAME_KEY, filename);
                        buf.append(extractor.getText(new IgnoreCloseInputStream(ais), map).getContent());
                        buf.append('\n');
                    } catch (final Exception e) {
                        if (logger.isDebugEnabled()) {
                            logger.debug("Exception in an internal extractor.", e);
                        }
                    }
                }
            }
        }
    } catch (final MaxLengthExceededException e) {
        throw e;
    } catch (final Exception e) {
        if (buf.length() == 0) {
            throw new ExtractException("Could not extract a content.", e);
        }
    } finally {
        CloseableUtil.closeQuietly(ais);
    }
    return buf.toString().trim();
}

Example 23 with ArchiveInputStream

use of org.apache.commons.compress.archivers.ArchiveInputStream in project fess-crawler by codelibs.

the class ZipExtractor method getText.

@Override
public ExtractData getText(final InputStream in, final Map<String, String> params) {
    if (in == null) {
        throw new CrawlerSystemException("The inputstream is null.");
    }
    final MimeTypeHelper mimeTypeHelper = getMimeTypeHelper();
    final ExtractorFactory extractorFactory = getExtractorFactory();
    final StringBuilder buf = new StringBuilder(1000);
    try (final ArchiveInputStream ais = archiveStreamFactory.createArchiveInputStream(in.markSupported() ? in : new BufferedInputStream(in))) {
        ZipArchiveEntry entry = null;
        long contentSize = 0;
        while ((entry = (ZipArchiveEntry) ais.getNextEntry()) != null) {
            contentSize += entry.getSize();
            if (maxContentSize != -1 && contentSize > maxContentSize) {
                throw new MaxLengthExceededException("Extracted size is " + contentSize + " > " + maxContentSize);
            }
            final String filename = entry.getName();
            final String mimeType = mimeTypeHelper.getContentType(null, filename);
            if (mimeType != null) {
                final Extractor extractor = extractorFactory.getExtractor(mimeType);
                if (extractor != null) {
                    try {
                        final Map<String, String> map = new HashMap<>();
                        map.put(ExtractData.RESOURCE_NAME_KEY, filename);
                        buf.append(extractor.getText(new IgnoreCloseInputStream(ais), map).getContent());
                        buf.append('\n');
                    } catch (final Exception e) {
                        if (logger.isDebugEnabled()) {
                            logger.debug("Exception in an internal extractor.", e);
                        }
                    }
                }
            }
        }
    } catch (final MaxLengthExceededException e) {
        throw e;
    } catch (final Exception e) {
        if (buf.length() == 0) {
            throw new ExtractException("Could not extract a content.", e);
        }
    }
    return new ExtractData(buf.toString().trim());
}

Example 24 with ArchiveInputStream

use of org.apache.commons.compress.archivers.ArchiveInputStream in project cloud-pipeline by epam.

the class ScanService method fetchLayer.

private void fetchLayer(ScanRequest.Layer layerToScan, File layerFolder) throws IOException {
    GZIPInputStream gzipInputStream = new GZIPInputStream(new BufferedInputStream(dockerRegistryService.getDockerLayerBlob(layerToScan)));
    LOGGER.debug("Unpack layer: " + layerToScan.getName() + " into: " + layerFolder.getAbsolutePath());
    try (ArchiveInputStream tarStream = new TarArchiveInputStream(gzipInputStream)) {
        ArchiveEntry entry;
        while ((entry = tarStream.getNextEntry()) != null) {
            String entryName = entry.getName();
            final File entryFile = new File(layerFolder, entryName);
            if (entry.isDirectory()) {
                Files.createDirectory(entryFile.toPath());
            } else {
                try (OutputStream out = new BufferedOutputStream(new FileOutputStream(entryFile))) {
                    IOUtils.copy(tarStream, out);
                }
            }
        }
        LOGGER.debug("Successfully unpack layer: " + layerToScan.getName());
    }
}

Example 25 with ArchiveInputStream

use of org.apache.commons.compress.archivers.ArchiveInputStream in project hutool by dromara.

the class StreamExtractor method extractInternal.

/**
 * 释放（解压）到指定目录
 *
 * @param targetDir 目标目录
 * @param filter    解压文件过滤器，用于指定需要释放的文件，null表示不过滤。当{@link Filter#accept(Object)}为true时释放。
 * @throws IOException IO异常
 */
private void extractInternal(File targetDir, Filter<ArchiveEntry> filter) throws IOException {
    Assert.isTrue(null != targetDir && ((false == targetDir.exists()) || targetDir.isDirectory()), "target must be dir.");
    final ArchiveInputStream in = this.in;
    ArchiveEntry entry;
    File outItemFile;
    while (null != (entry = in.getNextEntry())) {
        if (null != filter && false == filter.accept(entry)) {
            continue;
        }
        if (false == in.canReadEntryData(entry)) {
            // 无法读取的文件直接跳过
            continue;
        }
        outItemFile = FileUtil.file(targetDir, entry.getName());
        if (entry.isDirectory()) {
            // 创建对应目录
            // noinspection ResultOfMethodCallIgnored
            outItemFile.mkdirs();
        } else {
            FileUtil.writeFromStream(in, outItemFile, false);
        }
    }
}