Examples with DataMediaPair com.alibaba.otter.shared.common.model.config.data.DataMediaPair used on opensource projects

Example 46 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class ConfigHelper method findDataMediaPairByMediaId.

/**
 * 根据DataMedia id得到对应的DataMediaPair
 */
public static List<DataMediaPair> findDataMediaPairByMediaId(Pipeline pipeline, Long tid) {
    Assert.notNull(pipeline);
    List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
    for (DataMediaPair pair : pipeline.getPairs()) {
        if (pair.getSource().getId().equals(tid)) {
            pairs.add(pair);
        } else if (pair.getTarget().getId().equals(tid)) {
            pairs.add(pair);
        }
    }
    return pairs;
}

Example 47 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class OtterTransformerTest method test_rowData_oracle_mysql.

@Test
public void test_rowData_oracle_mysql() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
    DataMediaPair pair1 = new DataMediaPair();
    pair1.setId(1L);
    pair1.setPipelineId(pipeline.getId());
    pair1.setPullWeight(1L);
    pair1.setPushWeight(1L);
    DbDataMedia oracleMedia = getOracleMedia();
    oracleMedia.setId(1L);
    pair1.setSource(oracleMedia);
    DbDataMedia mysqlMedia = getMysqlMedia();
    pair1.setTarget(mysqlMedia);
    pairs.add(pair1);
    pipeline.setPairs(pairs);
    PipelineParameter param = new PipelineParameter();
    param.setSyncMode(SyncMode.ROW);
    pipeline.setParameters(param);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    EventData eventData = new EventData();
    eventData.setTableId(1L);
    eventData.setSchemaName("srf");
    eventData.setTableName("columns");
    eventData.setEventType(EventType.UPDATE);
    eventData.setExecuteTime(100L);
    eventData.getKeys().add(buildColumn("id", Types.NUMERIC, "1", true, false));
    eventData.getKeys().add(buildColumn("name", Types.VARCHAR, "ljh", true, false));
    eventData.getColumns().add(buildColumn("alias_name", Types.CHAR, "hello", false, false));
    eventData.getColumns().add(buildColumn("amount", Types.NUMERIC, "100.01", false, false));
    eventData.getColumns().add(buildColumn("text_b", Types.BLOB, "[116,101,120,116,95,98]", false, false));
    eventData.getColumns().add(buildColumn("text_c", Types.CLOB, "text_c", false, false));
    eventData.getColumns().add(buildColumn("curr_date", Types.DATE, "2011-01-01", false, false));
    eventData.getColumns().add(buildColumn("gmt_create", Types.DATE, "2011-01-01 11:11:11", false, false));
    eventData.getColumns().add(buildColumn("gmt_modify", Types.DATE, "2011-01-01 11:11:11", false, false));
    rowBatch.merge(eventData);
    Map<Class, BatchObject> batchs = otterTransformFactory.transform(rowBatch);
    RowBatch result = (RowBatch) batchs.get(EventData.class);
    want.number(result.getDatas().size()).isEqualTo(1);
}

Example 48 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class DbLoadActionTest method generatorDataMediaPairForMysql.

private List<DataMediaPair> generatorDataMediaPairForMysql(int count) {
    List<DataMediaPair> pairs = new ArrayList<DataMediaPair>();
    for (int i = 0; i < count; i++) {
        DataMediaPair pair = new DataMediaPair();
        int index = i + 1;
        pair.setId(Long.valueOf(index));
        pair.setPullWeight(count - Long.valueOf(index));
        pair.setPushWeight(count - Long.valueOf(index));
        DbDataMedia oracleMedia = getOracleMedia();
        oracleMedia.setId(2L);
        pair.setSource(oracleMedia);
        DbDataMedia mysqlMedia = getMysqlMedia();
        mysqlMedia.setId(1L);
        pair.setTarget(mysqlMedia);
        pairs.add(pair);
    }
    return pairs;
}

Example 49 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class DbLoadActionTest method test_db_load_oracle.

@Test
public void test_db_load_oracle() {
    ArbitrateConfigRegistry.regist(configClientService);
    dbLoadAction = (DbLoadAction) TestedObject.getSpringBeanFactory().getBean("dbLoadAction");
    final Channel channel = new Channel();
    channel.setId(1L);
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = generatorDataMediaPairForOracle(20);
    pipeline.setPairs(pairs);
    pipeline.getParameters().merge(new SystemParameter());
    pipeline.getParameters().merge(new ChannelParameter());
    // final Pipeline oppositePipeline = new Pipeline();
    // oppositePipeline.setId(101L);
    channel.setPipelines(Arrays.asList(pipeline));
    final Node currentNode = new Node();
    currentNode.setId(1L);
    new NonStrictExpectations() {

        {
            configClientService.findChannel(anyLong);
            returns(channel);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
            configClientService.currentNode();
            returns(currentNode);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    List<EventData> eventDatas = generatorEventDataForOracle(0, 20, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForOracle(10, 10, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForOracle(19, 1, EventType.DELETE);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    WeightController controller = new WeightController(1);
    dbLoadAction.load(rowBatch, controller);
}

Example 50 with DataMediaPair

use of com.alibaba.otter.shared.common.model.config.data.DataMediaPair in project otter by alibaba.

the class LocalFileLoaderActionTest method test_load_file.

@Test
public void test_load_file() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = generatorDataMediaPair(10);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    fileBatch.getFiles().addAll(generatorLocalFileData("fileLoad", 10));
    WeightController controller = new WeightController(1);
    fileLoadAction.load(fileBatch, new File(tmp + File.separator + OTTERLOAD), controller);
    File target = new File(tmp + File.separator + OTTERLOAD + "_loaded/");
    want.number(target.listFiles().length).isEqualTo(10);
    NioUtils.delete(target);
}