Examples with ArchiveInputStream org.apache.commons.compress.archivers.ArchiveInputStream used on opensource projects

Example 46 with ArchiveInputStream

use of org.apache.commons.compress.archivers.ArchiveInputStream in project BWAPI4J by OpenBW.

the class DummyDataUtils method readIntegerArrayFromArchiveFile.

public static int[] readIntegerArrayFromArchiveFile(final String archiveFilename, final String mapHash, final String regex) throws IOException {
    final InputStream inputStream = createInputStreamForDummyDataSet(archiveFilename);
    try (final ArchiveInputStream tarIn = new TarArchiveInputStream(new BZip2CompressorInputStream(inputStream));
        final BufferedReader buffer = new BufferedReader(new InputStreamReader(tarIn))) {
        final String mapShortHash = determineMapShortHash(mapHash);
        final ArchiveEntry nextEntry = getArchiveEntry(tarIn, mapShortHash);
        Assert.assertNotNull(nextEntry);
        final int[] read = buffer.lines().flatMap(line -> (Stream<String>) Stream.of(line.split(regex))).map(String::trim).mapToInt(Integer::parseInt).toArray();
        logger.debug("Read " + read.length + " values from " + archiveFilename);
        return read;
    }
}

Example 47 with ArchiveInputStream

use of org.apache.commons.compress.archivers.ArchiveInputStream in project agile-service by open-hand.

the class StaticFileCompressServiceImpl method unCompressedByApache.

/**
 * 可使用apache解压工具进行解压的流程一致，根据文件后缀名获取不同的压缩流
 *
 * @param staticFileCompress            解压参数
 * @param projectId                     项目id
 * @param organizationId                组织id
 * @param suffix                        文件后缀名
 * @param staticFileCompressHistoryList 解压操作历史记录
 * @throws IOException io错误
 */
private void unCompressedByApache(StaticFileCompressDTO staticFileCompress, Long projectId, Long organizationId, String suffix, List<StaticFileOperationHistoryDTO> staticFileCompressHistoryList) throws IOException {
    Long userId = DetailsHelper.getUserDetails().getUserId();
    StaticFileHeaderDTO update = new StaticFileHeaderDTO();
    update.setId(staticFileCompress.getId());
    int size = staticFileCompress.getSize();
    double process = 0.0;
    List<StaticFileLineDTO> lineList = new ArrayList<>();
    List<String> urls = new ArrayList<>();
    String prefixPath = staticFileCompress.getPrefixPath();
    try (BufferedInputStream bufferedInputStream = new BufferedInputStream(staticFileCompress.getIn());
        ArchiveInputStream in = getArchiveInputStream(bufferedInputStream, suffix, staticFileCompress.getEncode())) {
        ArchiveEntry entry;
        while (Objects.nonNull(entry = in.getNextEntry())) {
            int availableSize = bufferedInputStream.available();
            if (!entry.isDirectory() && in.canReadEntryData(entry)) {
                byte[] bytes = inputToByte(in);
                int newSize = bytes.length;
                // 跳过文件夹与不能读取数据的项
                if (entry.getName().contains(MACOSX) || entry.getName().contains(DS_STORE) || newSize <= 0) {
                    // 跳过冗余文件
                    continue;
                }
                // 文件上传
                String url = fileClient.uploadFile(organizationId, FileUploadBucket.AGILE_BUCKET.bucket(), null, getEntryFileName(entry.getName()), bytes);
                urls.add(url);
                String relativePath = filePathService.generateRelativePath(url);
                StaticFileLineDTO staticFileLine = new StaticFileLineDTO(projectId, organizationId, staticFileCompress.getId(), relativePath, dealRelativePath(entry.getName(), prefixPath));
                lineList.add(staticFileLine);
            }
            process = updateProcess(staticFileCompressHistoryList, staticFileCompress.getStaticFileCompressHistory(), size, (size - availableSize), process, staticFileCompress.getIssueId());
        }
        // 获取上传的文件信息
        List<FileDTO> files = fileClient.getFiles(organizationId, FileUploadBucket.AGILE_BUCKET.bucket(), urls);
        Map<String, FileDTO> fileMap = files.stream().collect(Collectors.toMap(file -> filePathService.generateRelativePath(file.getFileUrl()), file -> file));
        lineList.forEach(line -> {
            // 设置行的文件类型及其记录其他信息
            line.setId(snowflakeHelper.next());
            line.setCreatedBy(userId);
            line.setLastUpdatedBy(userId);
            line.setFileType(fileMap.get(line.getUrl()) != null ? fileMap.get(line.getUrl()).getFileType() : null);
        });
        staticFileLineMapper.batchInsert(lineList);
        updateHistoryStatus(staticFileCompress.getStaticFileCompressHistory(), SUCCESS);
        staticFileCompress.setStatus(SUCCESS);
        sendProcess(staticFileCompressHistoryList, staticFileCompress.getStaticFileCompressHistory().getUserId(), projectId, staticFileCompress.getIssueId());
    }
}

Example 48 with ArchiveInputStream

use of org.apache.commons.compress.archivers.ArchiveInputStream in project fess-crawler by codelibs.

the class TarExtractor method getTextInternal.

protected String getTextInternal(final InputStream in, final MimeTypeHelper mimeTypeHelper, final ExtractorFactory extractorFactory) {
    final StringBuilder buf = new StringBuilder(1000);
    ArchiveInputStream ais = null;
    try {
        ais = archiveStreamFactory.createArchiveInputStream("tar", in);
        TarArchiveEntry entry = null;
        long contentSize = 0;
        while ((entry = (TarArchiveEntry) ais.getNextEntry()) != null) {
            contentSize += entry.getSize();
            if (maxContentSize != -1 && contentSize > maxContentSize) {
                throw new MaxLengthExceededException("Extracted size is " + contentSize + " > " + maxContentSize);
            }
            final String filename = entry.getName();
            final String mimeType = mimeTypeHelper.getContentType(null, filename);
            if (mimeType != null) {
                final Extractor extractor = extractorFactory.getExtractor(mimeType);
                if (extractor != null) {
                    try {
                        final Map<String, String> map = new HashMap<>();
                        map.put(ExtractData.RESOURCE_NAME_KEY, filename);
                        buf.append(extractor.getText(new IgnoreCloseInputStream(ais), map).getContent());
                        buf.append('\n');
                    } catch (final Exception e) {
                        if (logger.isDebugEnabled()) {
                            logger.debug("Exception in an internal extractor.", e);
                        }
                    }
                }
            }
        }
    } catch (final MaxLengthExceededException e) {
        throw e;
    } catch (final Exception e) {
        if (buf.length() == 0) {
            throw new ExtractException("Could not extract a content.", e);
        }
    } finally {
        CloseableUtil.closeQuietly(ais);
    }
    return buf.toString().trim();
}

Example 49 with ArchiveInputStream

use of org.apache.commons.compress.archivers.ArchiveInputStream in project fess-crawler by codelibs.

the class ZipExtractor method getText.

@Override
public ExtractData getText(final InputStream in, final Map<String, String> params) {
    if (in == null) {
        throw new CrawlerSystemException("The inputstream is null.");
    }
    final MimeTypeHelper mimeTypeHelper = getMimeTypeHelper();
    final ExtractorFactory extractorFactory = getExtractorFactory();
    final StringBuilder buf = new StringBuilder(1000);
    try (final ArchiveInputStream ais = archiveStreamFactory.createArchiveInputStream(in.markSupported() ? in : new BufferedInputStream(in))) {
        ZipArchiveEntry entry = null;
        long contentSize = 0;
        while ((entry = (ZipArchiveEntry) ais.getNextEntry()) != null) {
            contentSize += entry.getSize();
            if (maxContentSize != -1 && contentSize > maxContentSize) {
                throw new MaxLengthExceededException("Extracted size is " + contentSize + " > " + maxContentSize);
            }
            final String filename = entry.getName();
            final String mimeType = mimeTypeHelper.getContentType(null, filename);
            if (mimeType != null) {
                final Extractor extractor = extractorFactory.getExtractor(mimeType);
                if (extractor != null) {
                    try {
                        final Map<String, String> map = new HashMap<>();
                        map.put(ExtractData.RESOURCE_NAME_KEY, filename);
                        buf.append(extractor.getText(new IgnoreCloseInputStream(ais), map).getContent());
                        buf.append('\n');
                    } catch (final Exception e) {
                        if (logger.isDebugEnabled()) {
                            logger.debug("Exception in an internal extractor.", e);
                        }
                    }
                }
            }
        }
    } catch (final MaxLengthExceededException e) {
        throw e;
    } catch (final Exception e) {
        if (buf.length() == 0) {
            throw new ExtractException("Could not extract a content.", e);
        }
    }
    return new ExtractData(buf.toString().trim());
}

Example 50 with ArchiveInputStream

use of org.apache.commons.compress.archivers.ArchiveInputStream in project selenium_java by sergueik.

the class FileExtractor method untarFolder.

private String untarFolder(InputStream compressedFileInputStream, String destinationFolder, List<String> possibleFilenames) throws IOException {
    String executablePath = "";
    ArchiveEntry currentFile;
    ArchiveInputStream archiveInputStream = new TarArchiveInputStream(compressedFileInputStream);
    CloseShieldInputStream notClosableArchiveInputStream = new CloseShieldInputStream(archiveInputStream);
    try {
        while ((currentFile = archiveInputStream.getNextEntry()) != null) {
            String name = currentFile.getName();
            name = this.handlePathCreation(name, destinationFolder);
            if (name.length() > 0) {
                String extractedFile = copyFileToDisk(notClosableArchiveInputStream, destinationFolder, name);
                for (String expectedFileName : possibleFilenames) {
                    if (extractedFile.endsWith(expectedFileName)) {
                        executablePath = extractedFile;
                    }
                }
            }
        }
    } finally {
        compressedFileInputStream.close();
        notClosableArchiveInputStream.close();
    }
    return executablePath;
}