Examples with ColumnPair com.alibaba.otter.shared.common.model.config.data.ColumnPair used on opensource projects

Example 6 with ColumnPair

use of com.alibaba.otter.shared.common.model.config.data.ColumnPair in project otter by alibaba.

the class DataColumnPairServiceImpl method listByDataMediaPairIds.

public Map<Long, List<ColumnPair>> listByDataMediaPairIds(Long... dataMediaPairIds) {
    Assert.assertNotNull(dataMediaPairIds);
    Map<Long, List<ColumnPair>> dataColumnPairs = new HashMap<Long, List<ColumnPair>>();
    try {
        List<DataColumnPairDO> dataColumnPairDos = dataColumnPairDao.listByDataMediaPairIds(dataMediaPairIds);
        if (dataColumnPairDos.isEmpty()) {
            logger.debug("DEBUG ## couldn't query any dataColumnPair, maybe hasn't create any dataColumnPair.");
            return dataColumnPairs;
        }
        for (DataColumnPairDO dataColumnPairDo : dataColumnPairDos) {
            List<ColumnPair> columnPairs = dataColumnPairs.get(dataColumnPairDo.getDataMediaPairId());
            if (columnPairs != null) {
                if (!columnPairs.contains(doToModel(dataColumnPairDo))) {
                    columnPairs.add(doToModel(dataColumnPairDo));
                }
            } else {
                columnPairs = new ArrayList<ColumnPair>();
                columnPairs.add(doToModel(dataColumnPairDo));
                dataColumnPairs.put(dataColumnPairDo.getDataMediaPairId(), columnPairs);
            }
        }
    } catch (Exception e) {
        logger.error("ERROR ## query dataColumnPair by dataMediaId:" + dataMediaPairIds + " has an exception!");
        throw new ManagerException(e);
    }
    return dataColumnPairs;
}

Example 7 with ColumnPair

use of com.alibaba.otter.shared.common.model.config.data.ColumnPair in project otter by alibaba.

the class RowDataTransformer method transform.

public EventData transform(EventData data, OtterTransformerContext context) {
    EventData result = new EventData();
    // 处理Table转化
    DataMedia dataMedia = context.getDataMediaPair().getTarget();
    result.setPairId(context.getDataMediaPair().getId());
    result.setTableId(dataMedia.getId());
    // 需要特殊处理下multi场景
    buildName(data, result, context.getDataMediaPair());
    result.setEventType(data.getEventType());
    result.setExecuteTime(data.getExecuteTime());
    result.setSyncConsistency(data.getSyncConsistency());
    result.setRemedy(data.isRemedy());
    result.setSyncMode(data.getSyncMode());
    result.setSize(data.getSize());
    result.setHint(data.getHint());
    result.setWithoutSchema(data.isWithoutSchema());
    if (data.getEventType().isDdl()) {
        // ddl不需要处理字段
        if (StringUtils.equalsIgnoreCase(result.getSchemaName(), data.getSchemaName()) && StringUtils.equalsIgnoreCase(result.getTableName(), data.getTableName())) {
            // 是否需要对ddl sql进行转化，暂时不支持异构，必须保证源表和目标表的名字相同
            result.setDdlSchemaName(data.getDdlSchemaName());
            result.setSql(data.getSql());
            return result;
        } else {
            throw new TransformException("no support ddl for [" + data.getSchemaName() + "." + data.getTableName() + "] to [" + result.getSchemaName() + "." + result.getTableName() + "] , sql :" + data.getSql());
        }
    }
    Multimap<String, String> translateColumnNames = HashMultimap.create();
    if (context.getDataMediaPair().getColumnPairMode().isInclude()) {
        // 只针对正向匹配进行名字映射，exclude不做处理
        List<ColumnPair> columnPairs = context.getDataMediaPair().getColumnPairs();
        for (ColumnPair columnPair : columnPairs) {
            translateColumnNames.put(columnPair.getSourceColumn().getName(), columnPair.getTargetColumn().getName());
        }
    }
    // 准备一下table meta
    DataMediaPair dataMediaPair = context.getDataMediaPair();
    boolean useTableTransform = context.getPipeline().getParameters().getUseTableTransform();
    boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn();
    TableInfoHolder tableHolder = null;
    if (useTableTransform || enableCompatibleMissColumn) {
        // 控制一下是否需要反查table
        // meta信息，如果同构数据库，完全没必要反查
        // 获取目标库的表信息
        DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), (DbMediaSource) dataMedia.getSource());
        Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName());
        tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn);
    }
    // 处理column转化
    List<EventColumn> otherColumns = translateColumns(result, data.getColumns(), context.getDataMediaPair(), translateColumnNames, tableHolder);
    translatePkColumn(result, data.getKeys(), data.getOldKeys(), otherColumns, context.getDataMediaPair(), translateColumnNames, tableHolder);
    result.setColumns(otherColumns);
    return result;
}

Example 8 with ColumnPair

use of com.alibaba.otter.shared.common.model.config.data.ColumnPair in project otter by alibaba.

the class AddColumnPairGroup method buildColumnPairFromDataMedia.

private List<ColumnPair> buildColumnPairFromDataMedia(Long dataMediaPairId, Long sourceMediaId, Long targetMediaId) {
    List<ColumnPair> columnPairs = new ArrayList<ColumnPair>();
    List<String> sourceColumns = dataMediaService.queryColumnByMediaId(sourceMediaId);
    List<String> targetColumns = dataMediaService.queryColumnByMediaId(targetMediaId);
    if (CollectionUtils.isNotEmpty(sourceColumns) && CollectionUtils.isNotEmpty(targetColumns)) {
        for (String sourceColumn : sourceColumns) {
            for (String targetColumn : targetColumns) {
                if (sourceColumn.equalsIgnoreCase(targetColumn)) {
                    ColumnPair temp = new ColumnPair(sourceColumn, targetColumn);
                    temp.setDataMediaPairId(dataMediaPairId);
                    columnPairs.add(temp);
                }
            }
        }
    }
    return columnPairs;
}

Example 9 with ColumnPair

use of com.alibaba.otter.shared.common.model.config.data.ColumnPair in project otter by alibaba.

the class ViewExtractor method columnFilter.

private List<EventColumn> columnFilter(List<EventColumn> eventColumns, List<ColumnPair> columnPairs, ColumnPairMode mode) {
    if (mode == null) {
        mode = ColumnPairMode.INCLUDE;
    }
    List<EventColumn> tempColumns = new ArrayList<EventColumn>();
    Map<String, ColumnPair> viewNames = new HashMap<String, ColumnPair>();
    for (ColumnPair columnPair : columnPairs) {
        viewNames.put(StringUtils.lowerCase(columnPair.getSourceColumn().getName()), columnPair);
    }
    for (EventColumn eventColumn : eventColumns) {
        if (mode.isInclude() && viewNames.containsKey(StringUtils.lowerCase(eventColumn.getColumnName()))) {
            // 正向匹配
            tempColumns.add(eventColumn);
        } else if (mode.isExclude() && !viewNames.containsKey(StringUtils.lowerCase(eventColumn.getColumnName()))) {
            // 逆向匹配
            tempColumns.add(eventColumn);
        }
    }
    return tempColumns;
}

Example 10 with ColumnPair

use of com.alibaba.otter.shared.common.model.config.data.ColumnPair in project otter by alibaba.

the class GroupExtractor method groupFilter.

private void groupFilter(EventData eventData, ColumnGroup columnGroup) {
    List<EventColumn> addColumns = new ArrayList<EventColumn>();
    // 判断一下是否存在字段组内字段的变更
    Set<String> updatedColumns = new HashSet<String>();
    Set<String> pks = new HashSet<String>();
    // 注意，这里只拿实际需要同步变更的字段
    for (EventColumn column : eventData.getUpdatedColumns()) {
        updatedColumns.add(column.getColumnName());
    }
    for (EventColumn pk : eventData.getKeys()) {
        pks.add(pk.getColumnName());
    }
    if (!CollectionUtils.isEmpty(eventData.getOldKeys())) {
        // 处理变更的主键
        int i = 0;
        for (EventColumn pk : eventData.getKeys()) {
            if (!StringUtils.equals(pk.getColumnValue(), eventData.getOldKeys().get(i).getColumnValue())) {
                updatedColumns.add(pk.getColumnName());
            }
            i++;
        }
    }
    if (containsInGroupColumn(updatedColumns, columnGroup.getColumnPairs())) {
        // 将变更的字段+变更的主键 去和 group字段进行交集处理
        for (ColumnPair columnPair : columnGroup.getColumnPairs()) {
            // 原谅我起这么长的变量名…
            boolean groupColumnHasInChangedColunms = false;
            // for (String columnName : updatedColumns) {
            for (EventColumn column : eventData.getColumns()) {
                if (StringUtils.equalsIgnoreCase(columnPair.getSourceColumn().getName(), column.getColumnName())) {
                    groupColumnHasInChangedColunms = true;
                    if (!column.isUpdate()) {
                        // 如果为非同步字段，强制修改为update=true进行数据同步
                        column.setUpdate(true);
                    }
                    break;
                }
            }
            if (!groupColumnHasInChangedColunms) {
                // 不存在对应的变更字段记录
                String columnName = columnPair.getSourceColumn().getName();
                if (!pks.contains(columnName)) {
                    // 只添加非主键的值到反查column，因为主键不需要反查
                    EventColumn addColumn = new EventColumn();
                    addColumn.setColumnName(columnPair.getSourceColumn().getName());
                    addColumn.setUpdate(true);
                    addColumns.add(addColumn);
                }
            }
        }
        if (!CollectionUtils.isEmpty(addColumns)) {
            // 字段去重
            // 添加不足的字段
            eventData.getColumns().addAll(addColumns);
            eventData.setSyncConsistency(SyncConsistency.MEDIA);
            return;
        }
    }
}