Examples with DataMedia com.alibaba.otter.shared.common.model.config.data.DataMedia used on opensource projects

Example 1 with DataMedia

use of com.alibaba.otter.shared.common.model.config.data.DataMedia in project otter by alibaba.

the class ColumnPairAction method doSave.

/**
     * 添加Channel
     * 
     * @param channelInfo
     * @param channelParameterInfo
     * @throws Exception
     */
public void doSave(@Param("dataMediaPairId") Long dataMediaPairId, @Param("submitKey") String submitKey, @Param("channelId") Long channelId, @Param("pipelineId") Long pipelineId, @Param("sourceMediaId") Long sourceMediaId, @Param("targetMediaId") Long targetMediaId, @FormGroup("columnPairInfo") Group columnPairInfo, @FormField(name = "formColumnPairError", group = "columnPairInfo") CustomErrors err, Navigator nav) throws Exception {
    String[] sourceColumns = columnPairInfo.getField("dltTarget_l").getStringValues();
    String[] targetColumns = columnPairInfo.getField("dltTarget_r").getStringValues();
    List<String> sourceColumnNames = new ArrayList<String>();
    List<String> targetColumnNames = new ArrayList<String>();
    for (String sourceColumn : sourceColumns) {
        sourceColumnNames.add(sourceColumn);
    }
    for (String targetColumn : targetColumns) {
        targetColumnNames.add(targetColumn);
    }
    DataMedia targetMedia = dataMediaPairService.findById(dataMediaPairId).getTarget();
    if (!targetMedia.getSource().getType().isNapoli() && sourceColumnNames.size() != targetColumnNames.size()) {
        err.setMessage("invalidColumnPair");
        return;
    }
    List<ColumnPair> columnPairsInDb = dataColumnPairService.listByDataMediaPairId(dataMediaPairId);
    List<ColumnPair> columnPairsTemp = new ArrayList<ColumnPair>();
    List<String> columnPairsNameSource = new ArrayList<String>();
    List<String> columnPairsNameTarget = new ArrayList<String>();
    List<ColumnPair> columnPairs = new ArrayList<ColumnPair>();
    if (targetMedia.getSource().getType().isNapoli()) {
        for (ColumnPair columnPair : columnPairsInDb) {
            for (String sourceColumnName : sourceColumnNames) {
                if (StringUtils.isEquals(columnPair.getSourceColumn().getName(), sourceColumnName)) {
                    columnPairsTemp.add(columnPair);
                    columnPairsNameSource.add(sourceColumnName);
                }
            }
        }
        // 要从数据库中删除这些columnPair
        columnPairsInDb.removeAll(columnPairsTemp);
        sourceColumnNames.removeAll(columnPairsNameSource);
        for (String columnName : sourceColumnNames) {
            ColumnPair columnPair = new ColumnPair();
            columnPair.setSourceColumn(new Column(columnName));
            columnPair.setDataMediaPairId(dataMediaPairId);
            columnPairs.add(columnPair);
        }
    } else if (targetMedia.getSource().getType().isMysql() || targetMedia.getSource().getType().isOracle()) {
        for (ColumnPair columnPair : columnPairsInDb) {
            int i = 0;
            for (String sourceColumnName : sourceColumnNames) {
                if (StringUtils.isEquals(columnPair.getSourceColumn().getName(), sourceColumnName) && StringUtils.isEquals(columnPair.getTargetColumn().getName(), targetColumnNames.get(i))) {
                    columnPairsTemp.add(columnPair);
                    columnPairsNameSource.add(sourceColumnName);
                    columnPairsNameTarget.add(targetColumnNames.get(i));
                }
                i++;
            }
        }
        // 要从数据库中删除这些columnPair
        columnPairsInDb.removeAll(columnPairsTemp);
        sourceColumnNames.removeAll(columnPairsNameSource);
        targetColumnNames.removeAll(columnPairsNameTarget);
        int i = 0;
        for (String columnName : sourceColumnNames) {
            ColumnPair columnPair = new ColumnPair();
            columnPair.setSourceColumn(new Column(columnName));
            columnPair.setTargetColumn(new Column(targetColumnNames.get(i)));
            columnPair.setDataMediaPairId(dataMediaPairId);
            columnPairs.add(columnPair);
            i++;
        }
    }
    for (ColumnPair columnPair : columnPairsInDb) {
        dataColumnPairService.remove(columnPair.getId());
    }
    dataColumnPairService.createBatch(columnPairs);
    if (submitKey.equals("保存")) {
        nav.redirectToLocation("dataMediaPairList.htm?pipelineId=" + pipelineId);
    } else if (submitKey.equals("下一步")) {
        nav.redirectToLocation("addColumnPairGroup.htm?dataMediaPairId=" + dataMediaPairId + "&channelId=" + channelId + "&pipelineId=" + pipelineId + "&sourceMediaId=" + sourceMediaId + "&targetMediaId=" + targetMediaId);
    }
}

Example 2 with DataMedia

use of com.alibaba.otter.shared.common.model.config.data.DataMedia in project otter by alibaba.

the class DataMediaPairAction method doBatchAdd.

/**
     * 批量添加DataMediaPair
     * 
     * @param dataMediaPairInfo
     * @throws Exception
     */
public void doBatchAdd(@FormGroup("batchDataMediaPairInfo") Group batchDataMediaPairInfo, @Param("pipelineId") Long pipelineId, @FormField(name = "formBatchDataMediaPairError", group = "batchDataMediaPairInfo") CustomErrors err, Navigator nav) throws Exception {
    String batchPairContent = batchDataMediaPairInfo.getField("batchPairContent").getStringValue();
    List<String> StringPairs = Arrays.asList(batchPairContent.split("\r\n"));
    try {
        for (String stringPair : StringPairs) {
            List<String> pairData = Arrays.asList(stringPair.split(","));
            if (pairData.size() < 4) {
                throw new ManagerException("[" + stringPair + "] the line not all parameters");
            }
            // build the pair source
            DataMedia sourceDataMedia = new DataMedia();
            DataMediaSource sourceDataMediaSource = dataMediaSourceService.findById(Long.parseLong(StringUtils.trimToNull(pairData.get(2))));
            sourceDataMedia.setNamespace(StringUtils.trimToNull(pairData.get(0)));
            sourceDataMedia.setName(StringUtils.trimToNull(pairData.get(1)));
            sourceDataMedia.setSource(sourceDataMediaSource);
            Long sourceMediaId = dataMediaService.createReturnId(sourceDataMedia);
            sourceDataMedia.setId(sourceMediaId);
            // build the pair target
            DataMedia targetDataMedia = new DataMedia();
            Long weight = 5L;
            if (StringUtils.isNumeric(pairData.get(3)) && pairData.size() <= 5) {
                // 如果是纯数字，那说明是简化配置模式
                DataMediaSource targetDataMediaSource = dataMediaSourceService.findById(Long.parseLong(StringUtils.trimToNull(pairData.get(3))));
                targetDataMedia.setNamespace(StringUtils.trimToNull(pairData.get(0)));
                targetDataMedia.setName(StringUtils.trimToNull(pairData.get(1)));
                targetDataMedia.setSource(targetDataMediaSource);
                Long targetMediaId = dataMediaService.createReturnId(targetDataMedia);
                targetDataMedia.setId(targetMediaId);
                if (pairData.size() >= 5) {
                    weight = Long.parseLong(StringUtils.trimToNull(pairData.get(4)));
                }
            } else {
                DataMediaSource targetDataMediaSource = dataMediaSourceService.findById(Long.parseLong(StringUtils.trimToNull(pairData.get(5))));
                targetDataMedia.setNamespace(StringUtils.trimToNull(pairData.get(3)));
                targetDataMedia.setName(StringUtils.trimToNull(pairData.get(4)));
                targetDataMedia.setSource(targetDataMediaSource);
                Long targetMediaId = dataMediaService.createReturnId(targetDataMedia);
                targetDataMedia.setId(targetMediaId);
                if (pairData.size() >= 7) {
                    weight = Long.parseLong(StringUtils.trimToNull(pairData.get(6)));
                }
            }
            // build the pair
            DataMediaPair dataMediaPair = new DataMediaPair();
            dataMediaPair.setSource(sourceDataMedia);
            dataMediaPair.setTarget(targetDataMedia);
            dataMediaPair.setPushWeight(weight);
            dataMediaPair.setPipelineId(pipelineId);
            dataMediaPairService.createIfNotExist(dataMediaPair);
        }
    } catch (Exception e) {
        err.setMessage("invalidBatchDataMediaPair");
        return;
    }
    nav.redirectToLocation("dataMediaPairList.htm?pipelineId=" + pipelineId);
}

Example 3 with DataMedia

use of com.alibaba.otter.shared.common.model.config.data.DataMedia in project otter by alibaba.

the class DataMediaPairAction method doNextToView.

/**
     * 选择视图同步
     * 
     * @param channelInfo
     * @param channelParameterInfo
     * @throws Exception
     */
public void doNextToView(@FormGroup("dataMediaPairInfo") Group dataMediaPairInfo, @FormField(name = "formDataMediaPairError", group = "dataMediaPairInfo") CustomErrors err, Navigator nav) throws Exception {
    DataMediaPair dataMediaPair = new DataMediaPair();
    DataMedia sourceDataMedia = new DataMedia();
    DataMedia targetDataMedia = new DataMedia();
    dataMediaPairInfo.setProperties(dataMediaPair);
    sourceDataMedia.setId(dataMediaPairInfo.getField("sourceDataMediaId").getLongValue());
    dataMediaPair.setSource(sourceDataMedia);
    targetDataMedia.setId(dataMediaPairInfo.getField("targetDataMediaId").getLongValue());
    dataMediaPair.setTarget(targetDataMedia);
    try {
        dataMediaPairService.create(dataMediaPair);
    } catch (RepeatConfigureException rce) {
        err.setMessage("invalidDataMediaPair");
        return;
    }
    nav.redirectToLocation("dataMediaPairList.htm?pipelineId=" + dataMediaPair.getPipelineId());
}

Example 4 with DataMedia

use of com.alibaba.otter.shared.common.model.config.data.DataMedia in project otter by alibaba.

the class DataMediaServiceImpl method listByCondition.

@Override
public List<DataMedia> listByCondition(Map condition) {
    List<DataMedia> dataMedias = new ArrayList<DataMedia>();
    try {
        List<DataMediaDO> dataMediaDos = dataMediaDao.listByCondition(condition);
        if (dataMediaDos.isEmpty()) {
            logger.debug("DEBUG ## couldn't query any dataMedias by the condition:" + JsonUtils.marshalToString(condition));
            return dataMedias;
        }
        dataMedias = doToModel(dataMediaDos);
    } catch (Exception e) {
        logger.error("ERROR ## query dataMedias by condition has an exception!");
        throw new ManagerException(e);
    }
    return dataMedias;
}

Example 5 with DataMedia

use of com.alibaba.otter.shared.common.model.config.data.DataMedia in project otter by alibaba.

the class RowDataTransformer method transform.

public EventData transform(EventData data, OtterTransformerContext context) {
    EventData result = new EventData();
    // 处理Table转化
    DataMedia dataMedia = context.getDataMediaPair().getTarget();
    result.setPairId(context.getDataMediaPair().getId());
    result.setTableId(dataMedia.getId());
    // 需要特殊处理下multi场景
    buildName(data, result, context.getDataMediaPair());
    result.setEventType(data.getEventType());
    result.setExecuteTime(data.getExecuteTime());
    result.setSyncConsistency(data.getSyncConsistency());
    result.setRemedy(data.isRemedy());
    result.setSyncMode(data.getSyncMode());
    result.setSize(data.getSize());
    result.setHint(data.getHint());
    result.setWithoutSchema(data.isWithoutSchema());
    if (data.getEventType().isDdl()) {
        // ddl不需要处理字段
        if (StringUtils.equalsIgnoreCase(result.getSchemaName(), data.getSchemaName()) && StringUtils.equalsIgnoreCase(result.getTableName(), data.getTableName())) {
            // 是否需要对ddl sql进行转化，暂时不支持异构，必须保证源表和目标表的名字相同
            result.setDdlSchemaName(data.getDdlSchemaName());
            result.setSql(data.getSql());
            return result;
        } else {
            throw new TransformException("no support ddl for [" + data.getSchemaName() + "." + data.getTableName() + "] to [" + result.getSchemaName() + "." + result.getTableName() + "] , sql :" + data.getSql());
        }
    }
    Multimap<String, String> translateColumnNames = HashMultimap.create();
    if (context.getDataMediaPair().getColumnPairMode().isInclude()) {
        // 只针对正向匹配进行名字映射，exclude不做处理
        List<ColumnPair> columnPairs = context.getDataMediaPair().getColumnPairs();
        for (ColumnPair columnPair : columnPairs) {
            translateColumnNames.put(columnPair.getSourceColumn().getName(), columnPair.getTargetColumn().getName());
        }
    }
    // 准备一下table meta
    DataMediaPair dataMediaPair = context.getDataMediaPair();
    boolean useTableTransform = context.getPipeline().getParameters().getUseTableTransform();
    boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn();
    TableInfoHolder tableHolder = null;
    if (useTableTransform || enableCompatibleMissColumn) {
        // 控制一下是否需要反查table
        // meta信息，如果同构数据库，完全没必要反查
        // 获取目标库的表信息
        DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), (DbMediaSource) dataMedia.getSource());
        Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName());
        tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn);
    }
    // 处理column转化
    List<EventColumn> otherColumns = translateColumns(result, data.getColumns(), context.getDataMediaPair(), translateColumnNames, tableHolder);
    translatePkColumn(result, data.getKeys(), data.getOldKeys(), otherColumns, context.getDataMediaPair(), translateColumnNames, tableHolder);
    result.setColumns(otherColumns);
    return result;
}