Examples with ColumnPair com.alibaba.otter.shared.common.model.config.data.ColumnPair used on opensource projects

Example 1 with ColumnPair

use of com.alibaba.otter.shared.common.model.config.data.ColumnPair in project otter by alibaba.

the class ColumnPairAction method doSave.

/**
 * 添加Channel
 *
 * @param channelInfo
 * @param channelParameterInfo
 * @throws Exception
 */
public void doSave(@Param("dataMediaPairId") Long dataMediaPairId, @Param("submitKey") String submitKey, @Param("channelId") Long channelId, @Param("pipelineId") Long pipelineId, @Param("sourceMediaId") Long sourceMediaId, @Param("targetMediaId") Long targetMediaId, @FormGroup("columnPairInfo") Group columnPairInfo, @FormField(name = "formColumnPairError", group = "columnPairInfo") CustomErrors err, Navigator nav) throws Exception {
    String[] sourceColumns = columnPairInfo.getField("dltTarget_l").getStringValues();
    String[] targetColumns = columnPairInfo.getField("dltTarget_r").getStringValues();
    List<String> sourceColumnNames = new ArrayList<String>();
    List<String> targetColumnNames = new ArrayList<String>();
    for (String sourceColumn : sourceColumns) {
        sourceColumnNames.add(sourceColumn);
    }
    for (String targetColumn : targetColumns) {
        targetColumnNames.add(targetColumn);
    }
    DataMedia targetMedia = dataMediaPairService.findById(dataMediaPairId).getTarget();
    if (!targetMedia.getSource().getType().isNapoli() && sourceColumnNames.size() != targetColumnNames.size()) {
        err.setMessage("invalidColumnPair");
        return;
    }
    List<ColumnPair> columnPairsInDb = dataColumnPairService.listByDataMediaPairId(dataMediaPairId);
    List<ColumnPair> columnPairsTemp = new ArrayList<ColumnPair>();
    List<String> columnPairsNameSource = new ArrayList<String>();
    List<String> columnPairsNameTarget = new ArrayList<String>();
    List<ColumnPair> columnPairs = new ArrayList<ColumnPair>();
    if (targetMedia.getSource().getType().isNapoli()) {
        for (ColumnPair columnPair : columnPairsInDb) {
            for (String sourceColumnName : sourceColumnNames) {
                if (StringUtils.isEquals(columnPair.getSourceColumn().getName(), sourceColumnName)) {
                    columnPairsTemp.add(columnPair);
                    columnPairsNameSource.add(sourceColumnName);
                }
            }
        }
        // 要从数据库中删除这些columnPair
        columnPairsInDb.removeAll(columnPairsTemp);
        sourceColumnNames.removeAll(columnPairsNameSource);
        for (String columnName : sourceColumnNames) {
            ColumnPair columnPair = new ColumnPair();
            columnPair.setSourceColumn(new Column(columnName));
            columnPair.setDataMediaPairId(dataMediaPairId);
            columnPairs.add(columnPair);
        }
    } else if (targetMedia.getSource().getType().isMysql() || targetMedia.getSource().getType().isOracle()) {
        for (ColumnPair columnPair : columnPairsInDb) {
            int i = 0;
            for (String sourceColumnName : sourceColumnNames) {
                if (StringUtils.isEquals(columnPair.getSourceColumn().getName(), sourceColumnName) && StringUtils.isEquals(columnPair.getTargetColumn().getName(), targetColumnNames.get(i))) {
                    columnPairsTemp.add(columnPair);
                    columnPairsNameSource.add(sourceColumnName);
                    columnPairsNameTarget.add(targetColumnNames.get(i));
                }
                i++;
            }
        }
        // 要从数据库中删除这些columnPair
        columnPairsInDb.removeAll(columnPairsTemp);
        sourceColumnNames.removeAll(columnPairsNameSource);
        targetColumnNames.removeAll(columnPairsNameTarget);
        int i = 0;
        for (String columnName : sourceColumnNames) {
            ColumnPair columnPair = new ColumnPair();
            columnPair.setSourceColumn(new Column(columnName));
            columnPair.setTargetColumn(new Column(targetColumnNames.get(i)));
            columnPair.setDataMediaPairId(dataMediaPairId);
            columnPairs.add(columnPair);
            i++;
        }
    }
    for (ColumnPair columnPair : columnPairsInDb) {
        dataColumnPairService.remove(columnPair.getId());
    }
    dataColumnPairService.createBatch(columnPairs);
    if (submitKey.equals("保存")) {
        nav.redirectToLocation("dataMediaPairList.htm?pipelineId=" + pipelineId);
    } else if (submitKey.equals("下一步")) {
        nav.redirectToLocation("addColumnPairGroup.htm?dataMediaPairId=" + dataMediaPairId + "&channelId=" + channelId + "&pipelineId=" + pipelineId + "&sourceMediaId=" + sourceMediaId + "&targetMediaId=" + targetMediaId);
    }
}

Example 2 with ColumnPair

use of com.alibaba.otter.shared.common.model.config.data.ColumnPair in project otter by alibaba.

the class DataColumnPairGroupServiceImpl method doToModel.

/*-------------------------------------------------------------*/
/**
 * 用于DO对象转化为Model对象
 */
private ColumnGroup doToModel(DataColumnPairGroupDO dataColumnPairGroupDo) {
    ColumnGroup columnGroup = new ColumnGroup();
    columnGroup.setId(dataColumnPairGroupDo.getId());
    List<ColumnPair> columnPairs = new ArrayList<ColumnPair>();
    if (StringUtils.isNotBlank(dataColumnPairGroupDo.getColumnPairContent())) {
        columnPairs = JsonUtils.unmarshalFromString(dataColumnPairGroupDo.getColumnPairContent(), new TypeReference<ArrayList<ColumnPair>>() {
        });
    }
    columnGroup.setColumnPairs(columnPairs);
    columnGroup.setDataMediaPairId(dataColumnPairGroupDo.getDataMediaPairId());
    columnGroup.setGmtCreate(dataColumnPairGroupDo.getGmtCreate());
    columnGroup.setGmtModified(dataColumnPairGroupDo.getGmtModified());
    return columnGroup;
}

Example 3 with ColumnPair

use of com.alibaba.otter.shared.common.model.config.data.ColumnPair in project otter by alibaba.

the class GroupExtractor method groupFilter.

private void groupFilter(EventData eventData, ColumnGroup columnGroup) {
    List<EventColumn> addColumns = new ArrayList<EventColumn>();
    // 判断一下是否存在字段组内字段的变更
    Set<String> updatedColumns = new HashSet<String>();
    Set<String> pks = new HashSet<String>();
    // 注意，这里只拿实际需要同步变更的字段
    for (EventColumn column : eventData.getUpdatedColumns()) {
        updatedColumns.add(column.getColumnName());
    }
    for (EventColumn pk : eventData.getKeys()) {
        pks.add(pk.getColumnName());
    }
    if (!CollectionUtils.isEmpty(eventData.getOldKeys())) {
        // 处理变更的主键
        int i = 0;
        for (EventColumn pk : eventData.getKeys()) {
            if (!StringUtils.equals(pk.getColumnValue(), eventData.getOldKeys().get(i).getColumnValue())) {
                updatedColumns.add(pk.getColumnName());
            }
            i++;
        }
    }
    if (containsInGroupColumn(updatedColumns, columnGroup.getColumnPairs())) {
        // 将变更的字段+变更的主键 去和 group字段进行交集处理
        for (ColumnPair columnPair : columnGroup.getColumnPairs()) {
            // 原谅我起这么长的变量名…
            boolean groupColumnHasInChangedColunms = false;
            // for (String columnName : updatedColumns) {
            for (EventColumn column : eventData.getColumns()) {
                if (StringUtils.equalsIgnoreCase(columnPair.getSourceColumn().getName(), column.getColumnName())) {
                    groupColumnHasInChangedColunms = true;
                    if (!column.isUpdate()) {
                        // 如果为非同步字段，强制修改为update=true进行数据同步
                        column.setUpdate(true);
                    }
                    break;
                }
            }
            if (!groupColumnHasInChangedColunms) {
                // 不存在对应的变更字段记录
                String columnName = columnPair.getSourceColumn().getName();
                if (!pks.contains(columnName)) {
                    // 只添加非主键的值到反查column，因为主键不需要反查
                    EventColumn addColumn = new EventColumn();
                    addColumn.setColumnName(columnPair.getSourceColumn().getName());
                    addColumn.setUpdate(true);
                    addColumns.add(addColumn);
                }
            }
        }
        if (!CollectionUtils.isEmpty(addColumns)) {
            // 字段去重
            // 添加不足的字段
            eventData.getColumns().addAll(addColumns);
            eventData.setSyncConsistency(SyncConsistency.MEDIA);
            return;
        }
    }
}

Example 4 with ColumnPair

use of com.alibaba.otter.shared.common.model.config.data.ColumnPair in project otter by alibaba.

the class ViewExtractor method columnFilter.

private List<EventColumn> columnFilter(List<EventColumn> eventColumns, List<ColumnPair> columnPairs, ColumnPairMode mode) {
    if (mode == null) {
        mode = ColumnPairMode.INCLUDE;
    }
    List<EventColumn> tempColumns = new ArrayList<EventColumn>();
    Map<String, ColumnPair> viewNames = new HashMap<String, ColumnPair>();
    for (ColumnPair columnPair : columnPairs) {
        viewNames.put(StringUtils.lowerCase(columnPair.getSourceColumn().getName()), columnPair);
    }
    for (EventColumn eventColumn : eventColumns) {
        if (mode.isInclude() && viewNames.containsKey(StringUtils.lowerCase(eventColumn.getColumnName()))) {
            // 正向匹配
            tempColumns.add(eventColumn);
        } else if (mode.isExclude() && !viewNames.containsKey(StringUtils.lowerCase(eventColumn.getColumnName()))) {
            // 逆向匹配
            tempColumns.add(eventColumn);
        }
    }
    return tempColumns;
}

Example 5 with ColumnPair

use of com.alibaba.otter.shared.common.model.config.data.ColumnPair in project otter by alibaba.

the class DataColumnPairServiceImpl method listByDataMediaPairIds.

public Map<Long, List<ColumnPair>> listByDataMediaPairIds(Long... dataMediaPairIds) {
    Assert.assertNotNull(dataMediaPairIds);
    Map<Long, List<ColumnPair>> dataColumnPairs = new HashMap<Long, List<ColumnPair>>();
    try {
        List<DataColumnPairDO> dataColumnPairDos = dataColumnPairDao.listByDataMediaPairIds(dataMediaPairIds);
        if (dataColumnPairDos.isEmpty()) {
            logger.debug("DEBUG ## couldn't query any dataColumnPair, maybe hasn't create any dataColumnPair.");
            return dataColumnPairs;
        }
        for (DataColumnPairDO dataColumnPairDo : dataColumnPairDos) {
            List<ColumnPair> columnPairs = dataColumnPairs.get(dataColumnPairDo.getDataMediaPairId());
            if (columnPairs != null) {
                if (!columnPairs.contains(doToModel(dataColumnPairDo))) {
                    columnPairs.add(doToModel(dataColumnPairDo));
                }
            } else {
                columnPairs = new ArrayList<ColumnPair>();
                columnPairs.add(doToModel(dataColumnPairDo));
                dataColumnPairs.put(dataColumnPairDo.getDataMediaPairId(), columnPairs);
            }
        }
    } catch (Exception e) {
        logger.error("ERROR ## query dataColumnPair by dataMediaId:" + dataMediaPairIds + " has an exception!");
        throw new ManagerException(e);
    }
    return dataColumnPairs;
}