Examples with FileLoadContext com.alibaba.otter.node.etl.load.loader.db.context.FileLoadContext used on opensource projects

Example 1 with FileLoadContext

use of com.alibaba.otter.node.etl.load.loader.db.context.FileLoadContext in project otter by alibaba.

the class FileLoadActionTest method testLoadWithLocal.

@Test
public void testLoadWithLocal() throws IOException {
    // 构造fileData使用的参数，fileDataStartIndex 决定着 pipeline 与 fileData 对应的关系（通过
    // dataMediaPair 的 id），
    // 以及 dataMediaPair 的 pushWeight
    final int fileDataStartIndex = 0;
    final int fileDataCount = 50;
    final Pipeline pipeline = buildPipeline(fileDataStartIndex, fileDataCount);
    final Channel channel = new Channel();
    new NonStrictExpectations() {

        {
            configClientService.findChannel(anyLong);
            returns(channel);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity id = buildIdentity(1L, 2L, 3L);
    FileBatch fileBatch = buildFileBatch(id);
    fileBatch.getFiles().addAll(buildFileDatas(null, EventType.INSERT, fileDataStartIndex, fileDataCount, true));
    WeightController controller = new WeightController(1);
    FileLoadContext context = fileLoadAction.load(fileBatch, ROOT_DIR, controller);
    want.object(context.getChannel()).isEqualTo(channel);
    want.object(context.getPipeline()).isEqualTo(pipeline);
    want.object(context.getPrepareDatas()).isEqualTo(fileBatch.getFiles());
    want.number(context.getProcessedDatas().size()).isEqualTo(fileBatch.getFiles().size());
}

Example 2 with FileLoadContext

use of com.alibaba.otter.node.etl.load.loader.db.context.FileLoadContext in project otter by alibaba.

the class FileLoadAction method buildContext.

private FileLoadContext buildContext(Identity identity) {
    FileLoadContext context = new FileLoadContext();
    context.setIdentity(identity);
    Channel channel = configClientService.findChannel(identity.getChannelId());
    Pipeline pipeline = configClientService.findPipeline(identity.getPipelineId());
    context.setChannel(channel);
    context.setPipeline(pipeline);
    return context;
}

Example 3 with FileLoadContext

use of com.alibaba.otter.node.etl.load.loader.db.context.FileLoadContext in project otter by alibaba.

the class FileLoadAction method load.

/**
     * 返回结果为已处理成功的记录
     */
public FileLoadContext load(FileBatch fileBatch, File rootDir, WeightController controller) {
    if (false == rootDir.exists()) {
        throw new LoadException(rootDir.getPath() + " is not exist");
    }
    FileLoadContext context = buildContext(fileBatch.getIdentity());
    context.setPrepareDatas(fileBatch.getFiles());
    boolean isDryRun = context.getPipeline().getParameters().isDryRun();
    try {
        // 复制成功的文件信息
        WeightBuckets<FileData> buckets = buildWeightBuckets(fileBatch.getIdentity(), fileBatch.getFiles());
        List<Long> weights = buckets.weights();
        controller.start(weights);
        // 处理数据
        for (int i = 0; i < weights.size(); i++) {
            Long weight = weights.get(i);
            controller.await(weight.intValue());
            if (logger.isInfoEnabled()) {
                logger.debug("##start load for weight:{}\n", weight);
            }
            // 处理同一个weight下的数据
            List<FileData> items = buckets.getItems(weight);
            if (context.getPipeline().getParameters().isDryRun()) {
                dryRun(context, items, rootDir);
            } else {
                moveFiles(context, items, rootDir);
            }
            controller.single(weight.intValue());
            if (logger.isInfoEnabled()) {
                logger.debug("##end load for weight:{}\n", weight);
            }
        }
        if (dump || isDryRun) {
            MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId()));
            logger.info(FileloadDumper.dumpContext("successed", context));
            MDC.remove(OtterConstants.splitPipelineLoadLogFileKey);
        }
    } catch (InterruptedException e) {
        Thread.currentThread().interrupt();
        if (dump || isDryRun) {
            MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId()));
            logger.info(FileloadDumper.dumpContext("error", context));
            MDC.remove(OtterConstants.splitPipelineLoadLogFileKey);
        }
    } catch (Exception e) {
        if (dump || isDryRun) {
            MDC.put(OtterConstants.splitPipelineLoadLogFileKey, String.valueOf(fileBatch.getIdentity().getPipelineId()));
            logger.info(FileloadDumper.dumpContext("error", context));
            MDC.remove(OtterConstants.splitPipelineLoadLogFileKey);
        }
        throw new LoadException(e);
    } finally {
        // 不论是否移动成功，删除临时目录
        NioUtils.delete(rootDir, 3);
    }
    return context;
}