Examples with LazyFileInputStream com.alibaba.otter.node.etl.common.pipe.impl.http.archive.LazyFileInputStream used on opensource projects

Example 1 with LazyFileInputStream

use of com.alibaba.otter.node.etl.common.pipe.impl.http.archive.LazyFileInputStream in project otter by alibaba.

the class ArchiveBeanIntegration method test_pack.

// @Test
public void test_pack() {
    ArchiveBean archiveBean = new ArchiveBean();
    try {
        archiveBean.afterPropertiesSet();
        archiveBean.setUseLocalFileMutliThread(false);
    } catch (Exception e1) {
        want.fail();
    }
    File file = new File("/tmp/otter/test");
    Collection<File> allFiles = FileUtils.listFiles(file, new String[] { "jpg" }, true);
    List<FileData> fileDatas = new ArrayList<FileData>();
    for (File files : allFiles) {
        FileData data = new FileData();
        // data.setPath("wsproduct_repository/product_sku/76/84/32/84/768432847_10.summ.jpg");
        data.setPath(StringUtils.substringAfter(files.getAbsolutePath(), "/tmp/otter/test"));
        fileDatas.add(data);
    }
    File archiveFile = new File("/tmp/otter/test.gzip");
    if (archiveFile.exists()) {
        archiveFile.delete();
    }
    boolean result = archiveBean.pack(archiveFile, fileDatas, new ArchiveRetriverCallback<FileData>() {

        public InputStream retrive(FileData source) {
            return new LazyFileInputStream(new File("/tmp/otter/test", source.getPath()));
        }
    });
    if (!result) {
        want.fail();
    }
}

Example 2 with LazyFileInputStream

use of com.alibaba.otter.node.etl.common.pipe.impl.http.archive.LazyFileInputStream in project otter by alibaba.

the class AttachmentHttpPipe method archiveFile.

// 处理对应的附件
private HttpPipeKey archiveFile(final FileBatch fileBatch) {
    // 处理构造对应的文件url
    String filename = buildFileName(fileBatch.getIdentity(), ClassUtils.getShortClassName(fileBatch.getClass()));
    File file = new File(htdocsDir, filename);
    // 压缩对应的文件数据
    List<FileData> fileDatas = fileBatch.getFiles();
    Pipeline pipeline = configClientService.findPipeline(fileBatch.getIdentity().getPipelineId());
    int poolSize = pipeline.getParameters().getFileLoadPoolSize();
    boolean useLocalFileMutliThread = pipeline.getParameters().getUseLocalFileMutliThread();
    ArchiveBean archiveBean = getArchiveBean();
    // 调整线程池大小
    archiveBean.adjustPoolSize(poolSize);
    // 设置是否启用local多线程同步
    archiveBean.setUseLocalFileMutliThread(useLocalFileMutliThread);
    boolean done = archiveBean.pack(file, fileDatas, new ArchiveRetriverCallback<FileData>() {

        public InputStream retrive(FileData fileData) {
            boolean miss = false;
            try {
                if (StringUtils.isNotEmpty(fileData.getNameSpace())) {
                    throw new RuntimeException(fileData + " is not support!");
                } else {
                    File source = new File(fileData.getPath());
                    if (source.exists() && source.isFile()) {
                        return new LazyFileInputStream(source);
                    } else {
                        miss = true;
                        return null;
                    }
                }
            } finally {
                if (miss && logger.isInfoEnabled()) {
                    MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId()));
                    logger.info(FileloadDumper.dumpMissFileDatas(fileBatch.getIdentity(), fileData));
                }
            }
        }
    });
    if (done == false) {
        // 直接返回
        return null;
    }
    HttpPipeKey key = new HttpPipeKey();
    key.setUrl(remoteUrlBuilder.getUrl(fileBatch.getIdentity().getPipelineId(), filename));
    key.setDataType(PipeDataType.FILE_BATCH);
    key.setIdentity(fileBatch.getIdentity());
    if (encrypt || pipeline.getParameters().getUseFileEncrypt()) {
        // 加密处理
        EncryptedData encryptedData = encryptFile(file);
        key.setKey(encryptedData.getKey());
        key.setCrc(encryptedData.getCrc());
    }
    return key;
}