Examples with BatchObject com.alibaba.otter.shared.etl.model.BatchObject used on opensource projects

Example 6 with BatchObject

use of com.alibaba.otter.shared.etl.model.BatchObject in project otter by alibaba.

the class OtterTransformerFactory method transform.

/**
     * 转化FileBatch对象
     */
public Map<Class, BatchObject> transform(FileBatch fileBatch) {
    final Identity identity = translateIdentity(fileBatch.getIdentity());
    List<FileData> fileDatas = fileBatch.getFiles();
    Map<Class, BatchObject> result = new HashMap<Class, BatchObject>();
    // 初始化默认值
    result.put(FileData.class, initBatchObject(identity, FileData.class));
    for (FileData fileData : fileDatas) {
        // 进行转化
        Long tableId = fileData.getTableId();
        Pipeline pipeline = configClientService.findPipeline(identity.getPipelineId());
        // 针对每个同步数据，可能会存在多路复制的情况
        List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline, tableId);
        for (DataMediaPair pair : dataMediaPairs) {
            if (!pair.getSource().getId().equals(tableId)) {
                // 过滤tableID不为源的同步
                continue;
            }
            Object item = fileDataTransformer.transform(fileData, new OtterTransformerContext(identity, pair, pipeline));
            if (item == null) {
                continue;
            }
            // 合并结果
            merge(identity, result, item);
        }
    }
    return result;
}

Example 7 with BatchObject

use of com.alibaba.otter.shared.etl.model.BatchObject in project otter by alibaba.

the class OtterTransformerFactory method merge.

// =============================== helper method
// ============================
// 将生成的item对象合并到结果对象中
private synchronized void merge(Identity identity, Map<Class, BatchObject> data, Object item) {
    Class clazz = item.getClass();
    BatchObject batchObject = data.get(clazz);
    // 初始化一下对象
    if (batchObject == null) {
        batchObject = initBatchObject(identity, clazz);
        data.put(clazz, batchObject);
    }
    // 进行merge处理
    if (batchObject instanceof RowBatch) {
        ((RowBatch) batchObject).merge((EventData) item);
    } else if (batchObject instanceof FileBatch) {
        ((FileBatch) batchObject).getFiles().add((FileData) item);
    } else {
        throw new TransformException("no support Data[" + clazz.getName() + "]");
    }
}