Examples with FileBatch com.alibaba.otter.shared.etl.model.FileBatch used on opensource projects

Example 11 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class DataBatchLoader method load.

public List<LoadContext> load(DbBatch data) {
    final RowBatch rowBatch = data.getRowBatch();
    final FileBatch fileBatch = data.getFileBatch();
    boolean existFileBatch = (rowBatch != null && !CollectionUtils.isEmpty(fileBatch.getFiles()) && data.getRoot() != null);
    boolean existRowBatch = (rowBatch != null && !CollectionUtils.isEmpty(rowBatch.getDatas()));
    int count = 0;
    List<RowBatch> rowBatchs = null;
    if (existRowBatch) {
        // 根据介质内容进行分类合并，每个介质一个载入通道
        rowBatchs = split(rowBatch);
        count += rowBatchs.size();
    }
    if (existFileBatch) {
        count += 1;
    }
    WeightController controller = new WeightController(count);
    List<Future> futures = new ArrayList<Future>();
    ExecutorCompletionService completionService = new ExecutorCompletionService(executorService);
    if (existFileBatch) {
        submitFileBatch(futures, completionService, fileBatch, data.getRoot(), controller);
    }
    if (existRowBatch) {
        submitRowBatch(futures, completionService, rowBatchs, controller);
    }
    // 先获取一下异步处理的结果，记录一下出错的index
    List<LoadContext> processedContexts = new ArrayList<LoadContext>();
    int index = 0;
    LoadException exception = null;
    while (index < futures.size()) {
        try {
            // 它也可能被打断
            Future future = completionService.take();
            future.get();
        } catch (InterruptedException e) {
            exception = new LoadException(e);
            break;
        } catch (ExecutionException e) {
            exception = new LoadException(e);
            break;
        }
        index++;
    }
    // 任何一个线程返回，出现了异常，就退出整个调度
    if (index < futures.size()) {
        // 小于代表有错误，需要对未完成的记录进行cancel操作，对已完成的结果进行收集，做重复录入过滤记录
        for (int errorIndex = 0; errorIndex < futures.size(); errorIndex++) {
            Future future = futures.get(errorIndex);
            if (future.isDone()) {
                try {
                    LoadContext loadContext = (LoadContext) future.get();
                    if (loadContext instanceof DbLoadContext) {
                        // 做一下出错处理，记录到store中
                        dbInterceptor.error((DbLoadContext) loadContext);
                    }
                } catch (InterruptedException e) {
                // ignore
                } catch (ExecutionException e) {
                // ignore
                } catch (Exception e) {
                    logger.error("interceptor process error failed", e);
                }
            } else {
                // 对未完成的进行取消
                future.cancel(true);
            }
        }
    } else {
        for (int i = 0; i < futures.size(); i++) {
            // 收集一下正确处理完成的结果
            Future future = futures.get(i);
            try {
                LoadContext loadContext = (LoadContext) future.get();
                if (loadContext instanceof DbLoadContext) {
                    processedContexts.add((DbLoadContext) loadContext);
                }
            } catch (InterruptedException e) {
            // ignore
            } catch (ExecutionException e) {
            // ignore
            }
        }
    }
    if (exception != null) {
        throw exception;
    } else {
        return processedContexts;
    }
}

Example 12 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class FileBatchConflictDetectServiceImpl method detect.

public FileBatch detect(FileBatch fileBatch, Long targetNodeId) {
    FileConflictDetectEvent event = new FileConflictDetectEvent();
    event.setFileBatch(fileBatch);
    if (isLocal(targetNodeId)) {
        return onFileConflictDetect(event);
    } else {
        // 调用远程
        return (FileBatch) nodeCommmunicationClient.call(targetNodeId, event);
    }
}

Example 13 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class FileLoadActionTest method testWithOutRootDir.

@Test
public void testWithOutRootDir() throws Exception {
    File rootDir = new File("/null");
    Identity id = buildIdentity(1L, 2L, 3L);
    FileBatch fileBatch = buildFileBatch(id);
    fileBatch.getFiles().addAll(buildFileDatas("ns_", EventType.INSERT, 0, 20, false));
    try {
        fileLoadAction.load(fileBatch, rootDir, null);
    } catch (Exception e) {
        // expect for LoadException
        if (e instanceof LoadException) {
            return;
        }
        throw e;
    }
    want.fail("unreachable code.");
}

Example 14 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class LocalFileLoaderActionTest method test_load_file.

@Test
public void test_load_file() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = generatorDataMediaPair(10);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    fileBatch.getFiles().addAll(generatorLocalFileData("fileLoad", 10));
    WeightController controller = new WeightController(1);
    fileLoadAction.load(fileBatch, new File(tmp + File.separator + OTTERLOAD), controller);
    File target = new File(tmp + File.separator + OTTERLOAD + "_loaded/");
    want.number(target.listFiles().length).isEqualTo(10);
    NioUtils.delete(target);
}

Example 15 with FileBatch

use of com.alibaba.otter.shared.etl.model.FileBatch in project otter by alibaba.

the class HttpPipeIntegration method test_attachment.

@Test
public void test_attachment() {
    final Node currentNode = new Node();
    currentNode.setId(1L);
    currentNode.setIp("127.0.0.1");
    currentNode.setParameters(new NodeParameter());
    final Pipeline pipeline = new Pipeline();
    pipeline.getParameters().setRetriever(RetrieverType.ARIA2C);
    // mock一下
    new NonStrictExpectations() {

        {
            configClientService.currentNode();
            returns(currentNode);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    File localFile = new File(tmp, "httpPipeTest.jpg");
    FileData localFileData = new FileData();
    localFileData.setEventType(EventType.INSERT);
    localFileData.setPath(localFile.getPath());
    fileBatch.getFiles().add(localFileData);
    try {
        byte[] data = getBlock(10 * 1024);
        NioUtils.write(data, localFile);
        HttpPipeKey key = attachmentHttpPipe.put(fileBatch);
        File target = attachmentHttpPipe.get(key);
        byte[] getbytes = NioUtils.read(new File(target, localFile.getPath()));
        check(data, getbytes);
    } catch (IOException e) {
        want.fail();
    } finally {
        NioUtils.delete(localFile);
    }
}