Search in sources :

Example 1 with ZipEntry

use of de.schlichtherle.util.zip.ZipEntry in project otter by alibaba.

the class ArchiveBean method doPack.

/**
     * 执行压缩
     */
@SuppressWarnings("resource")
private boolean doPack(final File targetArchiveFile, List<FileData> fileDatas, final ArchiveRetriverCallback<FileData> callback) {
    // 首先判断下对应的目标文件是否存在,如存在则执行删除
    if (true == targetArchiveFile.exists() && false == NioUtils.delete(targetArchiveFile, 3)) {
        throw new ArchiveException(String.format("[%s] exist and delete failed", targetArchiveFile.getAbsolutePath()));
    }
    boolean exist = false;
    ZipOutputStream zipOut = null;
    Set<String> entryNames = new HashSet<String>();
    // 下载成功的任务列表
    BlockingQueue<Future<ArchiveEntry>> queue = new LinkedBlockingQueue<Future<ArchiveEntry>>();
    ExecutorCompletionService completionService = new ExecutorCompletionService(executor, queue);
    final File targetDir = new File(targetArchiveFile.getParentFile(), FilenameUtils.getBaseName(targetArchiveFile.getPath()));
    try {
        // 创建一个临时目录
        FileUtils.forceMkdir(targetDir);
        zipOut = new ZipOutputStream(new BufferedOutputStream(new FileOutputStream(targetArchiveFile)));
        zipOut.setLevel(Deflater.BEST_SPEED);
        // 进行并发压缩处理
        for (final FileData fileData : fileDatas) {
            if (fileData.getEventType().isDelete()) {
                // 忽略delete类型的数据打包,因为只需直接在目标进行删除
                continue;
            }
            String namespace = fileData.getNameSpace();
            String path = fileData.getPath();
            boolean isLocal = StringUtils.isBlank(namespace);
            String entryName = null;
            if (true == isLocal) {
                entryName = FilenameUtils.getPath(path) + FilenameUtils.getName(path);
            } else {
                entryName = namespace + File.separator + path;
            }
            // 过滤一些重复的文件数据同步
            if (entryNames.contains(entryName) == false) {
                entryNames.add(entryName);
            } else {
                continue;
            }
            final String name = entryName;
            if (true == isLocal && !useLocalFileMutliThread) {
                // 采用串行处理,不走临时文件
                queue.add(new DummyFuture(new ArchiveEntry(name, callback.retrive(fileData))));
            } else {
                completionService.submit(new Callable<ArchiveEntry>() {

                    public ArchiveEntry call() throws Exception {
                        // 处理下异常,可能失败
                        InputStream input = null;
                        OutputStream output = null;
                        try {
                            input = callback.retrive(fileData);
                            if (input instanceof LazyFileInputStream) {
                                // 获取原始的stream
                                input = ((LazyFileInputStream) input).getInputSteam();
                            }
                            if (input != null) {
                                File tmp = new File(targetDir, name);
                                // 尝试创建父路径
                                NioUtils.create(tmp.getParentFile(), false, 3);
                                output = new FileOutputStream(tmp);
                                // 拷贝到文件
                                NioUtils.copy(input, output);
                                return new ArchiveEntry(name, new File(targetDir, name));
                            } else {
                                return new ArchiveEntry(name);
                            }
                        } finally {
                            IOUtils.closeQuietly(input);
                            IOUtils.closeQuietly(output);
                        }
                    }
                });
            }
        }
        for (int i = 0; i < entryNames.size(); i++) {
            // 读入流
            ArchiveEntry input = null;
            InputStream stream = null;
            try {
                input = queue.take().get();
                if (input == null) {
                    continue;
                }
                stream = input.getStream();
                if (stream == null) {
                    continue;
                }
                if (stream instanceof LazyFileInputStream) {
                    // 获取原始的stream
                    stream = ((LazyFileInputStream) stream).getInputSteam();
                }
                exist = true;
                zipOut.putNextEntry(new ZipEntry(input.getName()));
                // 输出到压缩流中
                NioUtils.copy(stream, zipOut);
                zipOut.closeEntry();
            } finally {
                IOUtils.closeQuietly(stream);
            }
        }
        if (exist) {
            zipOut.finish();
        }
    } catch (Exception e) {
        throw new ArchiveException(e);
    } finally {
        IOUtils.closeQuietly(zipOut);
        try {
            // 删除临时目录
            FileUtils.deleteDirectory(targetDir);
        } catch (IOException e) {
        // ignore
        }
    }
    return exist;
}
Also used : FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) BufferedOutputStream(java.io.BufferedOutputStream) OutputStream(java.io.OutputStream) FileOutputStream(java.io.FileOutputStream) ZipOutputStream(de.schlichtherle.util.zip.ZipOutputStream) ZipEntry(de.schlichtherle.util.zip.ZipEntry) ExecutorCompletionService(java.util.concurrent.ExecutorCompletionService) IOException(java.io.IOException) LinkedBlockingQueue(java.util.concurrent.LinkedBlockingQueue) TimeoutException(java.util.concurrent.TimeoutException) IOException(java.io.IOException) FileNotFoundException(java.io.FileNotFoundException) ExecutionException(java.util.concurrent.ExecutionException) ZipOutputStream(de.schlichtherle.util.zip.ZipOutputStream) FileOutputStream(java.io.FileOutputStream) Future(java.util.concurrent.Future) ZipFile(de.schlichtherle.util.zip.ZipFile) File(java.io.File) BufferedOutputStream(java.io.BufferedOutputStream) FileData(com.alibaba.otter.shared.etl.model.FileData) HashSet(java.util.HashSet)

Example 2 with ZipEntry

use of de.schlichtherle.util.zip.ZipEntry in project otter by alibaba.

the class ArchiveBean method unpack.

public List<File> unpack(File archiveFile, File targetDir) throws ArchiveException {
    // 首先判断下对应的目标文件是否存在,如存在则执行删除
    if (false == archiveFile.exists()) {
        throw new ArchiveException(String.format("[%s] not exist", archiveFile.getAbsolutePath()));
    }
    if (false == targetDir.exists() && false == NioUtils.create(targetDir, false, 3)) {
        throw new ArchiveException(String.format("[%s] not exist and create failed", targetDir.getAbsolutePath()));
    }
    List<File> result = new ArrayList<File>();
    ZipFile zipFile = null;
    try {
        zipFile = new ZipFile(archiveFile);
        Enumeration entries = zipFile.entries();
        while (entries.hasMoreElements()) {
            // entry
            ZipEntry entry = (ZipEntry) entries.nextElement();
            String entryName = entry.getName();
            // target
            File targetFile = new File(targetDir, entryName);
            // 尝试创建父路径
            NioUtils.create(targetFile.getParentFile(), false, 3);
            InputStream input = null;
            OutputStream output = null;
            try {
                output = new FileOutputStream(targetFile);
                input = zipFile.getInputStream(entry);
                NioUtils.copy(input, output);
            } finally {
                IOUtils.closeQuietly(input);
                IOUtils.closeQuietly(output);
            }
            result.add(targetFile);
        }
    } catch (Exception e) {
        throw new ArchiveException(e);
    } finally {
        if (zipFile != null) {
            try {
                zipFile.close();
            } catch (IOException ex) {
            }
        }
    }
    return result;
}
Also used : Enumeration(java.util.Enumeration) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) ZipEntry(de.schlichtherle.util.zip.ZipEntry) BufferedOutputStream(java.io.BufferedOutputStream) OutputStream(java.io.OutputStream) FileOutputStream(java.io.FileOutputStream) ZipOutputStream(de.schlichtherle.util.zip.ZipOutputStream) ArrayList(java.util.ArrayList) IOException(java.io.IOException) TimeoutException(java.util.concurrent.TimeoutException) IOException(java.io.IOException) FileNotFoundException(java.io.FileNotFoundException) ExecutionException(java.util.concurrent.ExecutionException) ZipFile(de.schlichtherle.util.zip.ZipFile) FileOutputStream(java.io.FileOutputStream) ZipFile(de.schlichtherle.util.zip.ZipFile) File(java.io.File)

Aggregations

ZipEntry (de.schlichtherle.util.zip.ZipEntry)2 ZipFile (de.schlichtherle.util.zip.ZipFile)2 ZipOutputStream (de.schlichtherle.util.zip.ZipOutputStream)2 BufferedOutputStream (java.io.BufferedOutputStream)2 File (java.io.File)2 FileInputStream (java.io.FileInputStream)2 FileNotFoundException (java.io.FileNotFoundException)2 FileOutputStream (java.io.FileOutputStream)2 IOException (java.io.IOException)2 InputStream (java.io.InputStream)2 OutputStream (java.io.OutputStream)2 ExecutionException (java.util.concurrent.ExecutionException)2 TimeoutException (java.util.concurrent.TimeoutException)2 FileData (com.alibaba.otter.shared.etl.model.FileData)1 ArrayList (java.util.ArrayList)1 Enumeration (java.util.Enumeration)1 HashSet (java.util.HashSet)1 ExecutorCompletionService (java.util.concurrent.ExecutorCompletionService)1 Future (java.util.concurrent.Future)1 LinkedBlockingQueue (java.util.concurrent.LinkedBlockingQueue)1