Examples with DataMediaPair com.alibaba.otter.shared.common.model.config.data.DataMediaPair used on opensource projects

Example 11 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class OtterTransformerTest method test_fileData.

@Test
public void test_fileData() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
    DataMediaPair pair1 = new DataMediaPair();
    pair1.setId(1L);
    pair1.setPipelineId(pipeline.getId());
    pair1.setPullWeight(1L);
    pair1.setPushWeight(1L);
    DbDataMedia oracleMedia = getOracleMedia();
    oracleMedia.setId(1L);
    pair1.setSource(oracleMedia);
    DbDataMedia mysqlMedia = getMysqlMedia();
    pair1.setTarget(mysqlMedia);
    pairs.add(pair1);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    File localFile = new File("/tmp", "httpPipeTest.jpg");
    FileData localFileData = new FileData();
    localFileData.setTableId(1L);
    localFileData.setPairId(1L);
    localFileData.setPath(localFile.getPath());
    fileBatch.getFiles().add(localFileData);
    Map<Class, BatchObject> batchs = otterTransformFactory.transform(fileBatch);
    FileBatch result = (FileBatch) batchs.get(FileData.class);
    want.number(result.getFiles().size()).isEqualTo(1);
}

Example 12 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class RowDataTransformer method transform.

public EventData transform(EventData data, OtterTransformerContext context) {
    EventData result = new EventData();
    // 处理Table转化
    DataMedia dataMedia = context.getDataMediaPair().getTarget();
    result.setPairId(context.getDataMediaPair().getId());
    result.setTableId(dataMedia.getId());
    // 需要特殊处理下multi场景
    buildName(data, result, context.getDataMediaPair());
    result.setEventType(data.getEventType());
    result.setExecuteTime(data.getExecuteTime());
    result.setSyncConsistency(data.getSyncConsistency());
    result.setRemedy(data.isRemedy());
    result.setSyncMode(data.getSyncMode());
    result.setSize(data.getSize());
    result.setHint(data.getHint());
    result.setWithoutSchema(data.isWithoutSchema());
    if (data.getEventType().isDdl()) {
        // ddl不需要处理字段
        if (StringUtils.equalsIgnoreCase(result.getSchemaName(), data.getSchemaName()) && StringUtils.equalsIgnoreCase(result.getTableName(), data.getTableName())) {
            // 是否需要对ddl sql进行转化，暂时不支持异构，必须保证源表和目标表的名字相同
            result.setDdlSchemaName(data.getDdlSchemaName());
            result.setSql(data.getSql());
            return result;
        } else {
            throw new TransformException("no support ddl for [" + data.getSchemaName() + "." + data.getTableName() + "] to [" + result.getSchemaName() + "." + result.getTableName() + "] , sql :" + data.getSql());
        }
    }
    Multimap<String, String> translateColumnNames = HashMultimap.create();
    if (context.getDataMediaPair().getColumnPairMode().isInclude()) {
        // 只针对正向匹配进行名字映射，exclude不做处理
        List<ColumnPair> columnPairs = context.getDataMediaPair().getColumnPairs();
        for (ColumnPair columnPair : columnPairs) {
            translateColumnNames.put(columnPair.getSourceColumn().getName(), columnPair.getTargetColumn().getName());
        }
    }
    // 准备一下table meta
    DataMediaPair dataMediaPair = context.getDataMediaPair();
    boolean useTableTransform = context.getPipeline().getParameters().getUseTableTransform();
    boolean enableCompatibleMissColumn = context.getPipeline().getParameters().getEnableCompatibleMissColumn();
    TableInfoHolder tableHolder = null;
    if (useTableTransform || enableCompatibleMissColumn) {
        // 控制一下是否需要反查table
        // meta信息，如果同构数据库，完全没必要反查
        // 获取目标库的表信息
        DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), (DbMediaSource) dataMedia.getSource());
        Table table = dbDialect.findTable(result.getSchemaName(), result.getTableName());
        tableHolder = new TableInfoHolder(table, useTableTransform, enableCompatibleMissColumn);
    }
    // 处理column转化
    List<EventColumn> otherColumns = translateColumns(result, data.getColumns(), context.getDataMediaPair(), translateColumnNames, tableHolder);
    translatePkColumn(result, data.getKeys(), data.getOldKeys(), otherColumns, context.getDataMediaPair(), translateColumnNames, tableHolder);
    result.setColumns(otherColumns);
    return result;
}

Example 13 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class DatabaseExtractorTest method test_global_row.

@Test
public void test_global_row() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    pipeline.getParameters().setSyncMode(SyncMode.ROW);
    // 设置为全局
    pipeline.getParameters().setSyncConsistency(SyncConsistency.MEDIA);
    int start = RandomUtils.nextInt();
    int count = 10;
    List<DataMediaPair> pairs = getDataMediaPairForMysql(start, count);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(100L);
            returns(pipeline);
        }
    };
    // 构造数据
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    for (int tableId = start; tableId < start + count; tableId++) {
        for (int i = start; i < start + count; i++) {
            EventData eventData = getEventData(tableId, i);
            eventData.setSchemaName("srf");
            eventData.setTableName("columns");
            rowBatch.merge(eventData);
        }
    }
    databaseExtractor.extract(new DbBatch(rowBatch));
    want.number(rowBatch.getDatas().size()).isEqualTo(count);
}

Example 14 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class DatabaseExtractorTest method getDataMediaPairForOracle.

private List<DataMediaPair> getDataMediaPairForOracle(long tableId, int count) {
    List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
    for (int i = 0; i < count; i++) {
        DataMediaPair pair = new DataMediaPair();
        pair.setId(Long.valueOf(i));
        pair.setPullWeight(1L);
        pair.setPushWeight(1L);
        DbDataMedia oracleMedia = getOracleMedia();
        oracleMedia.setId(tableId + i);
        pair.setSource(oracleMedia);
        DbDataMedia mysqlMedia = getMysqlMedia();
        mysqlMedia.setId(tableId + i + count);
        pair.setTarget(mysqlMedia);
        pairs.add(pair);
    }
    return pairs;
}

Example 15 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class FreedomExtractorTest method test_mysql.

@Test
public void test_mysql() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    int start = RandomUtils.nextInt();
    int count = 10;
    List<DataMediaPair> pairs = getDataMediaPairForMysql(start, count);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(100L);
            returns(pipeline);
        }
    };
    // 构造数据
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    for (int tableId = start; tableId < start + count; tableId++) {
        for (int i = start; i < start + count; i++) {
            EventData eventData = getEventData(tableId, i);
            eventData.setSchemaName("retl");
            eventData.setTableName("retl_buffer");
            rowBatch.merge(eventData);
        }
    }
    DbBatch dbBatch = new DbBatch(rowBatch);
    freedomExtractor.extract(dbBatch);
    want.collection(dbBatch.getRowBatch().getDatas()).sizeEq(count * count);
}