Examples with ColumnPairMode com.alibaba.otter.shared.common.model.config.data.ColumnPairMode used on opensource projects

Example 1 with ColumnPairMode

use of com.alibaba.otter.shared.common.model.config.data.ColumnPairMode in project otter by alibaba.

the class ViewExtractor method extract.

@Override
public void extract(DbBatch dbBatch) throws ExtractException {
    Assert.notNull(dbBatch);
    Assert.notNull(dbBatch.getRowBatch());
    Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
    List<DataMediaPair> dataMediaPairs = pipeline.getPairs();
    /**
         * Key = TableId<br>
         * Value = a List of this tableId's column need to sync<br>
         */
    Map<Long, List<ColumnPair>> viewColumnPairs = new HashMap<Long, List<ColumnPair>>();
    Map<Long, ColumnPairMode> viewColumnPairModes = new HashMap<Long, ColumnPairMode>();
    for (DataMediaPair dataMediaPair : dataMediaPairs) {
        List<ColumnPair> columnPairs = dataMediaPair.getColumnPairs();
        // 设置ColumnPairMode
        viewColumnPairModes.put(dataMediaPair.getSource().getId(), dataMediaPair.getColumnPairMode());
        // 如果没有columnPairs，则默认全字段同步，不做处理
        if (!CollectionUtils.isEmpty(columnPairs)) {
            viewColumnPairs.put(dataMediaPair.getSource().getId(), columnPairs);
        }
    }
    List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
    // 使用set，提升remove时的查找速度
    Set<EventData> removeDatas = new HashSet<EventData>();
    for (EventData eventData : eventDatas) {
        if (eventData.getEventType().isDdl()) {
            continue;
        }
        List<ColumnPair> columns = viewColumnPairs.get(eventData.getTableId());
        if (!CollectionUtils.isEmpty(columns)) {
            // 组装需要同步的Column
            ColumnPairMode mode = viewColumnPairModes.get(eventData.getTableId());
            eventData.setColumns(columnFilter(eventData.getColumns(), columns, mode));
            eventData.setKeys(columnFilter(eventData.getKeys(), columns, mode));
            if (!CollectionUtils.isEmpty(eventData.getOldKeys())) {
                eventData.setOldKeys(columnFilter(eventData.getOldKeys(), columns, mode));
            }
            if (CollectionUtils.isEmpty(eventData.getKeys())) {
                // 无主键，报错
                throw new ExtractException(String.format("eventData after viewExtractor has no pks , pls check! identity:%s, new eventData:%s", dbBatch.getRowBatch().getIdentity().toString(), eventData.toString()));
            }
            // update: 过滤后如果无字段(变更需要同步)和主键变更，则可以忽略之，避免sql语法错误
            if (eventData.getEventType().isUpdate() && (CollectionUtils.isEmpty(eventData.getColumns()) || CollectionUtils.isEmpty(eventData.getUpdatedColumns())) && CollectionUtils.isEmpty(eventData.getOldKeys())) {
                // 过滤之后无字段需要同步，并且不存在主键变更同步，则忽略该记录
                removeDatas.add(eventData);
            }
        }
    }
    if (!CollectionUtils.isEmpty(removeDatas)) {
        eventDatas.removeAll(removeDatas);
    }
}