Examples with ColumnGroup com.alibaba.otter.shared.common.model.config.data.ColumnGroup used on opensource projects

Example 1 with ColumnGroup

use of com.alibaba.otter.shared.common.model.config.data.ColumnGroup in project otter by alibaba.

the class DataColumnPairGroupServiceImpl method listByDataMediaPairIds.

@Override
public Map<Long, List<ColumnGroup>> listByDataMediaPairIds(Long... dataMediaPairIds) {
    Assert.assertNotNull(dataMediaPairIds);
    Map<Long, List<ColumnGroup>> dataColumnGroups = new HashMap<Long, List<ColumnGroup>>();
    try {
        List<DataColumnPairGroupDO> dataColumnPairGroupDos = dataColumnPairGroupDao.ListByDataMediaPairIds(dataMediaPairIds);
        if (CollectionUtils.isEmpty(dataColumnPairGroupDos)) {
            logger.debug("DEBUG ## couldn't query any dataColumnPairGroup, maybe hasn't create any dataColumnPairGroup.");
            return dataColumnGroups;
        }
        for (DataColumnPairGroupDO dataColumnPairGroupDo : dataColumnPairGroupDos) {
            List<ColumnGroup> columnGroups = dataColumnGroups.get(dataColumnPairGroupDo.getDataMediaPairId());
            if (columnGroups != null) {
                if (!columnGroups.contains(doToModel(dataColumnPairGroupDo))) {
                    columnGroups.add(doToModel(dataColumnPairGroupDo));
                }
            } else {
                columnGroups = new ArrayList<ColumnGroup>();
                columnGroups.add(doToModel(dataColumnPairGroupDo));
                dataColumnGroups.put(dataColumnPairGroupDo.getDataMediaPairId(), columnGroups);
            }
        }
    } catch (Exception e) {
        logger.error("ERROR ## query dataColumnPairGroup by dataMediaId:" + dataMediaPairIds + " has an exception!");
        throw new ManagerException(e);
    }
    return dataColumnGroups;
}

Example 2 with ColumnGroup

use of com.alibaba.otter.shared.common.model.config.data.ColumnGroup in project otter by alibaba.

the class DataColumnPairGroupServiceImpl method doToModel.

/*-------------------------------------------------------------*/
/**
     * 用于DO对象转化为Model对象
     */
private ColumnGroup doToModel(DataColumnPairGroupDO dataColumnPairGroupDo) {
    ColumnGroup columnGroup = new ColumnGroup();
    columnGroup.setId(dataColumnPairGroupDo.getId());
    List<ColumnPair> columnPairs = new ArrayList<ColumnPair>();
    if (StringUtils.isNotBlank(dataColumnPairGroupDo.getColumnPairContent())) {
        columnPairs = JsonUtils.unmarshalFromString(dataColumnPairGroupDo.getColumnPairContent(), new TypeReference<ArrayList<ColumnPair>>() {
        });
    }
    columnGroup.setColumnPairs(columnPairs);
    columnGroup.setDataMediaPairId(dataColumnPairGroupDo.getDataMediaPairId());
    columnGroup.setGmtCreate(dataColumnPairGroupDo.getGmtCreate());
    columnGroup.setGmtModified(dataColumnPairGroupDo.getGmtModified());
    return columnGroup;
}

Example 3 with ColumnGroup

use of com.alibaba.otter.shared.common.model.config.data.ColumnGroup in project otter by alibaba.

the class GroupExtractor method extract.

@Override
public void extract(DbBatch dbBatch) throws ExtractException {
    Assert.notNull(dbBatch);
    Assert.notNull(dbBatch.getRowBatch());
    Pipeline pipeline = getPipeline(dbBatch.getRowBatch().getIdentity().getPipelineId());
    List<DataMediaPair> dataMediaPairs = pipeline.getPairs();
    /**
         * Key = TableId<br>
         * Value = a List of this tableId's column need to sync<br>
         */
    Map<Long, List<ColumnGroup>> groupColumns = new HashMap<Long, List<ColumnGroup>>();
    for (DataMediaPair dataMediaPair : dataMediaPairs) {
        List<ColumnGroup> columnGroups = dataMediaPair.getColumnGroups();
        if (!CollectionUtils.isEmpty(columnGroups)) {
            groupColumns.put(dataMediaPair.getSource().getId(), columnGroups);
        }
    }
    List<EventData> eventDatas = dbBatch.getRowBatch().getDatas();
    for (EventData eventData : eventDatas) {
        if (eventData.getEventType().isDdl()) {
            continue;
        }
        List<ColumnGroup> columnGroups = groupColumns.get(eventData.getTableId());
        if (!CollectionUtils.isEmpty(columnGroups)) {
            for (ColumnGroup columnGroup : columnGroups) {
                if (columnGroup != null && !CollectionUtils.isEmpty(columnGroup.getColumnPairs())) {
                    groupFilter(eventData, columnGroup);
                }
            }
        }
    }
}

Example 4 with ColumnGroup

use of com.alibaba.otter.shared.common.model.config.data.ColumnGroup in project otter by alibaba.

the class DataMediaPairServiceImpl method doToModel.

private List<DataMediaPair> doToModel(List<DataMediaPairDO> dataMediaPairDos) {
    List<Long> dataMediaPairIds = new ArrayList<Long>();
    for (DataMediaPairDO dataMediaPairDo : dataMediaPairDos) {
        dataMediaPairIds.add(dataMediaPairDo.getId());
    }
    Map<Long, List<ColumnPair>> columnPairMap = dataColumnPairService.listByDataMediaPairIds(dataMediaPairIds.toArray(new Long[dataMediaPairIds.size()]));
    Map<Long, List<ColumnGroup>> columnPairGroupMap = dataColumnPairGroupService.listByDataMediaPairIds(dataMediaPairIds.toArray(new Long[dataMediaPairIds.size()]));
    List<DataMediaPair> dataMediaPairs = new ArrayList<DataMediaPair>();
    for (DataMediaPairDO dataMediaPairDo : dataMediaPairDos) {
        List<ColumnPair> columnPairs = columnPairMap.get(dataMediaPairDo.getId()) == null ? new ArrayList<ColumnPair>() : columnPairMap.get(dataMediaPairDo.getId());
        List<ColumnGroup> columnGroups = columnPairGroupMap.get(dataMediaPairDo.getId()) == null ? new ArrayList<ColumnGroup>() : columnPairGroupMap.get(dataMediaPairDo.getId());
        dataMediaPairs.add(doToModel(dataMediaPairDo, columnPairs, columnGroups));
    }
    return dataMediaPairs;
}

Example 5 with ColumnGroup

use of com.alibaba.otter.shared.common.model.config.data.ColumnGroup in project otter by alibaba.

the class ColumnPairGroupAction method doSave.

public void doSave(@Param("dataMediaPairId") Long dataMediaPairId, @Param("submitKey") String submitKey, @Param("pipelineId") Long pipelineId, @FormGroup("columnPairGroupInfo") Group columnPairGroupInfo, @FormField(name = "formColumnPairGroupError", group = "columnPairGroupInfo") CustomErrors err, Navigator nav) throws Exception {
    String[] columnPairStrings = columnPairGroupInfo.getField("groupResult").getStringValues();
    ColumnGroup columnGroup = new ColumnGroup();
    List<ColumnPair> columnPairs = new ArrayList<ColumnPair>();
    for (String columnPairString : columnPairStrings) {
        ColumnPair columnPair = new ColumnPair();
        String[] temp = columnPairString.split(COLON);
        columnPair.setSourceColumn(new Column(temp[0]));
        columnPair.setTargetColumn(new Column(temp[1]));
        columnPair.setDataMediaPairId(dataMediaPairId);
        columnPairs.add(columnPair);
    }
    columnGroup.setColumnPairs(columnPairs);
    columnGroup.setDataMediaPairId(dataMediaPairId);
    dataColumnPairGroupService.removeByDataMediaPairId(dataMediaPairId);
    dataColumnPairGroupService.create(columnGroup);
    nav.redirectToLocation("dataMediaPairList.htm?pipelineId=" + pipelineId);
}