Examples with ArchiveBean com.alibaba.otter.node.etl.common.pipe.impl.http.archive.ArchiveBean used on opensource projects

Example 1 with ArchiveBean

use of com.alibaba.otter.node.etl.common.pipe.impl.http.archive.ArchiveBean in project otter by alibaba.

the class ArchiveBeanIntegration method test_pack.

// @Test
public void test_pack() {
    ArchiveBean archiveBean = new ArchiveBean();
    try {
        archiveBean.afterPropertiesSet();
        archiveBean.setUseLocalFileMutliThread(false);
    } catch (Exception e1) {
        want.fail();
    }
    File file = new File("/tmp/otter/test");
    Collection<File> allFiles = FileUtils.listFiles(file, new String[] { "jpg" }, true);
    List<FileData> fileDatas = new ArrayList<FileData>();
    for (File files : allFiles) {
        FileData data = new FileData();
        // data.setPath("wsproduct_repository/product_sku/76/84/32/84/768432847_10.summ.jpg");
        data.setPath(StringUtils.substringAfter(files.getAbsolutePath(), "/tmp/otter/test"));
        fileDatas.add(data);
    }
    File archiveFile = new File("/tmp/otter/test.gzip");
    if (archiveFile.exists()) {
        archiveFile.delete();
    }
    boolean result = archiveBean.pack(archiveFile, fileDatas, new ArchiveRetriverCallback<FileData>() {

        public InputStream retrive(FileData source) {
            return new LazyFileInputStream(new File("/tmp/otter/test", source.getPath()));
        }
    });
    if (!result) {
        want.fail();
    }
}

Example 2 with ArchiveBean

use of com.alibaba.otter.node.etl.common.pipe.impl.http.archive.ArchiveBean in project otter by alibaba.

the class ArchiveBeanIntegration method test_unpack.

@Test
public void test_unpack() {
    ArchiveBean archiveBean = new ArchiveBean();
    try {
        archiveBean.afterPropertiesSet();
        archiveBean.setUseLocalFileMutliThread(false);
    } catch (Exception e1) {
        want.fail();
    }
    File archiveFile = new File("/tmp/otter/test.gzip");
    // for (File archiveFile : archiveFiles.listFiles()) {
    // if (archiveFile.isDirectory()) {
    // continue;
    // }
    // 
    // if (!"FileBatch-2013-03-07-16-27-05-245-369-3209577.gzip".equals(archiveFile.getName())) {
    // continue;
    // }
    System.out.println("start unpack : " + archiveFile.getName());
    File targetFile = new File("/tmp/otter", FilenameUtils.removeExtension(archiveFile.getName()));
    archiveBean.unpack(archiveFile, targetFile);
// }
}

Example 3 with ArchiveBean

use of com.alibaba.otter.node.etl.common.pipe.impl.http.archive.ArchiveBean in project otter by alibaba.

the class ArchiveBeanTest method test_simple.

@Test
public void test_simple() {
    File[] files = new File[10];
    List<FileData> fileDatas = new ArrayList<FileData>();
    File archiveFile = new File(tmp, "pack.zip");
    File unpack = new File(tmp, "unpack");
    ArchiveBean archiveBean = new ArchiveBean();
    try {
        archiveBean.afterPropertiesSet();
    } catch (Exception e1) {
        want.fail();
    }
    try {
        // for (int i = 0; i < 10; i++) {
        // files[i] = new File(tmp, "archiveTest_" + i + ".txt");
        // datas[i] = getBlock((i + 1) * 1024);
        // NioUtils.write(datas[i], files[i]);
        // 
        // FileData filedata = new FileData();
        // filedata.setPath(files[i].getPath());
        // fileDatas.add(filedata);
        // }
        archiveBean.pack(archiveFile, fileDatas, new ArchiveRetriverCallback<FileData>() {

            public InputStream retrive(FileData source) {
                try {
                    return new FileInputStream(new File(source.getPath()));
                } catch (FileNotFoundException e) {
                    e.printStackTrace();
                    want.fail();
                }
                return null;
            }
        });
        // 开始解压
        List<File> result = archiveBean.unpack(archiveFile, unpack);
        want.bool(result.size() == fileDatas.size());
    // File dir = new File(unpack, archiveFile.getParent());
    // File[] unpackFiles = dir.listFiles();
    // 
    // List<File> unpackFilesList = Arrays.asList(unpackFiles);
    // Collections.sort(unpackFilesList); // 排序一下
    // for (int i = 0; i < unpackFilesList.size(); i++) {
    // byte[] data = NioUtils.read(unpackFilesList.get(i));
    // check(data, datas[i]);
    // }
    } catch (Exception e) {
        want.fail();
    } finally {
        for (int i = 0; i < files.length; i++) {
            NioUtils.delete(files[i]);
        }
        NioUtils.delete(archiveFile);
        NioUtils.delete(unpack);
    }
}

Example 4 with ArchiveBean

use of com.alibaba.otter.node.etl.common.pipe.impl.http.archive.ArchiveBean in project otter by alibaba.

the class AttachmentHttpPipe method archiveFile.

// 处理对应的附件
private HttpPipeKey archiveFile(final FileBatch fileBatch) {
    // 处理构造对应的文件url
    String filename = buildFileName(fileBatch.getIdentity(), ClassUtils.getShortClassName(fileBatch.getClass()));
    File file = new File(htdocsDir, filename);
    // 压缩对应的文件数据
    List<FileData> fileDatas = fileBatch.getFiles();
    Pipeline pipeline = configClientService.findPipeline(fileBatch.getIdentity().getPipelineId());
    int poolSize = pipeline.getParameters().getFileLoadPoolSize();
    boolean useLocalFileMutliThread = pipeline.getParameters().getUseLocalFileMutliThread();
    ArchiveBean archiveBean = getArchiveBean();
    // 调整线程池大小
    archiveBean.adjustPoolSize(poolSize);
    // 设置是否启用local多线程同步
    archiveBean.setUseLocalFileMutliThread(useLocalFileMutliThread);
    boolean done = archiveBean.pack(file, fileDatas, new ArchiveRetriverCallback<FileData>() {

        public InputStream retrive(FileData fileData) {
            boolean miss = false;
            try {
                if (StringUtils.isNotEmpty(fileData.getNameSpace())) {
                    throw new RuntimeException(fileData + " is not support!");
                } else {
                    File source = new File(fileData.getPath());
                    if (source.exists() && source.isFile()) {
                        return new LazyFileInputStream(source);
                    } else {
                        miss = true;
                        return null;
                    }
                }
            } finally {
                if (miss && logger.isInfoEnabled()) {
                    MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId()));
                    logger.info(FileloadDumper.dumpMissFileDatas(fileBatch.getIdentity(), fileData));
                }
            }
        }
    });
    if (done == false) {
        // 直接返回
        return null;
    }
    HttpPipeKey key = new HttpPipeKey();
    key.setUrl(remoteUrlBuilder.getUrl(fileBatch.getIdentity().getPipelineId(), filename));
    key.setDataType(PipeDataType.FILE_BATCH);
    key.setIdentity(fileBatch.getIdentity());
    if (encrypt || pipeline.getParameters().getUseFileEncrypt()) {
        // 加密处理
        EncryptedData encryptedData = encryptFile(file);
        key.setKey(encryptedData.getKey());
        key.setCrc(encryptedData.getCrc());
    }
    return key;
}