Examples with DataMediaPair com.alibaba.otter.shared.common.model.config.data.DataMediaPair used on opensource projects

Example 21 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class DataMediaPairList method execute.

public void execute(@Param("pipelineId") Long pipelineId, Context context) throws Exception {
    // Pipeline pipeline = pipelineService.findById(pipelineId);
    Channel channel = channelService.findByPipelineId(pipelineId);
    List<DataMediaPair> dataMediaPairs = dataMediaPairService.listByPipelineId(pipelineId);
    Map<Long, TableStat> tableStatMap = new HashMap<Long, TableStat>(dataMediaPairs.size(), 1f);
    List<TableStat> tableStats = tableStatService.listTableStat(pipelineId);
    for (DataMediaPair dataMediaPair : dataMediaPairs) {
        int flag = 0;
        for (TableStat tableStat : tableStats) {
            if (dataMediaPair.getId().equals(tableStat.getDataMediaPairId())) {
                tableStatMap.put(dataMediaPair.getId(), tableStat);
                flag = 1;
                break;
            }
        }
        if (flag == 0) {
            TableStat tableStat = new TableStat();
            tableStat.setFileSize(0L);
            tableStat.setFileCount(0L);
            tableStat.setDeleteCount(0L);
            tableStat.setUpdateCount(0L);
            tableStat.setInsertCount(0L);
            // tableStat.setGmtModified(dataMediaPair.getGmtModified());
            tableStatMap.put(dataMediaPair.getId(), tableStat);
        }
    }
    context.put("dataMediaPairs", dataMediaPairs);
    // 通过PipelineId不能获取到Channel状态，所以需要传递Channel对象
    context.put("channel", channel);
    context.put("pipelineId", pipelineId);
    context.put("tableStatMap", tableStatMap);
}

Example 22 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class DbLoadAction method buildWeightBuckets.

/**
     * 构建基于weight权重分组的item集合列表
     */
private WeightBuckets<EventData> buildWeightBuckets(DbLoadContext context, List<EventData> datas) {
    WeightBuckets<EventData> buckets = new WeightBuckets<EventData>();
    for (EventData data : datas) {
        // 获取对应的weight
        DataMediaPair pair = ConfigHelper.findDataMediaPair(context.getPipeline(), data.getPairId());
        buckets.addItem(pair.getPushWeight(), data);
    }
    return buckets;
}

Example 23 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class FileExtractor method doFileExtract.

/**
     * 返回这批变更数据对应的FileInfo.
     * 
     * @param rowBatch
     * @return
     */
private List<FileData> doFileExtract(RowBatch rowBatch) {
    List<FileData> fileDatas = new ArrayList<FileData>();
    // 处理数据
    Pipeline pipeline = getPipeline(rowBatch.getIdentity().getPipelineId());
    List<EventData> eventDatas = rowBatch.getDatas();
    for (EventData eventData : eventDatas) {
        if (eventData.getEventType().isDdl()) {
            continue;
        }
        List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline, eventData.getTableId());
        if (dataMediaPairs == null) {
            throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId() + " dataMediaPair is null,please check");
        }
        for (DataMediaPair dataMediaPair : dataMediaPairs) {
            if (dataMediaPair.getResolverData() == null || dataMediaPair.getResolverData().getExtensionDataType() == null || (dataMediaPair.getResolverData().getExtensionDataType().isClazz() && StringUtils.isBlank(dataMediaPair.getResolverData().getClazzPath())) || (dataMediaPair.getResolverData().getExtensionDataType().isSource() && StringUtils.isBlank(dataMediaPair.getResolverData().getSourceText()))) {
                continue;
            }
            FileResolver fileResolver = null;
            if (dataMediaPair.getResolverData() != null) {
                fileResolver = extensionFactory.getExtension(FileResolver.class, dataMediaPair.getResolverData());
            } else {
                continue;
            }
            if (fileResolver == null) {
                throw new ExtractException("ERROR ## the dataMediaId = " + eventData.getTableId() + " the fileResolver className  = " + dataMediaPair.getResolverData().getClazzPath() + " is null ,please check the class");
            }
            if (fileResolver instanceof RemoteDirectoryFetcherAware) {
                RemoteDirectoryFetcherAware remoteDirectoryFetcherAware = (RemoteDirectoryFetcherAware) fileResolver;
                remoteDirectoryFetcherAware.setRemoteDirectoryFetcher(arandaRemoteDirectoryFetcher);
            }
            List<FileData> singleRowFileDatas = getSingleRowFileInfos(dataMediaPair.getId(), fileResolver, eventData);
            // 做一下去重处理
            for (FileData data : singleRowFileDatas) {
                if (!fileDatas.contains(data)) {
                    fileDatas.add(data);
                }
            }
        }
    }
    // 判断是否需要进行图片重复同步检查
    if (pipeline.getParameters().getFileDetect()) {
        doFileDetectCollector(pipeline, fileDatas);
    }
    return fileDatas;
}

Example 24 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class GroupExtractor method extract.

@Override
public void extract(DbBatch dbBatch) throws ExtractException {
    Assert.notNull(dbBatch);
    Assert.notNull(dbBatch.getRowBatch());
    Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
    List<DataMediaPair> dataMediaPairs = pipeline.getPairs();
    /**
         * Key = TableId<br>
         * Value = a List of this tableId's column need to sync<br>
         */
    Map<Long, List<ColumnGroup>> groupColumns = new HashMap<Long, List<ColumnGroup>>();
    for (DataMediaPair dataMediaPair : dataMediaPairs) {
        List<ColumnGroup> columnGroups = dataMediaPair.getColumnGroups();
        if (!CollectionUtils.isEmpty(columnGroups)) {
            groupColumns.put(dataMediaPair.getSource().getId(), columnGroups);
        }
    }
    List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
    for (EventData eventData : eventDatas) {
        if (eventData.getEventType().isDdl()) {
            continue;
        }
        List<ColumnGroup> columnGroups = groupColumns.get(eventData.getTableId());
        if (!CollectionUtils.isEmpty(columnGroups)) {
            for (ColumnGroup columnGroup : columnGroups) {
                if (columnGroup != null && !CollectionUtils.isEmpty(columnGroup.getColumnPairs())) {
                    groupFilter(eventData, columnGroup);
                }
            }
        }
    }
}

Example 25 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class FileLoadAction method buildWeightBuckets.

/**
     * 构建基于weight权重分组的item集合列表
     */
private WeightBuckets<FileData> buildWeightBuckets(Identity identity, List<FileData> datas) {
    WeightBuckets<FileData> buckets = new WeightBuckets<FileData>();
    for (FileData data : datas) {
        // 获取对应的weight
        DataMediaPair pair = ConfigHelper.findDataMediaPair(getPipeline(identity), data.getPairId());
        buckets.addItem(pair.getPushWeight(), data);
    }
    return buckets;
}