Search in sources :

Example 41 with EventData

use of com.alibaba.otter.shared.etl.model.EventData in project otter by alibaba.

the class FreedomExtractorTest method test_oracle.

@Test
public void test_oracle() {
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    int start = RandomUtils.nextInt();
    int count = 10;
    List<DataMediaPair> pairs = getDataMediaPairForOracle(start, count);
    pipeline.setPairs(pairs);
    new NonStrictExpectations() {

        {
            configClientService.findPipeline(100L);
            returns(pipeline);
        }
    };
    // 构造数据
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    for (int tableId = start; tableId < start + count; tableId++) {
        for (int i = start; i < start + count; i++) {
            EventData eventData = getEventData(tableId, i);
            eventData.setSchemaName("retl");
            eventData.setTableName("retl_buffer");
            rowBatch.merge(eventData);
        }
    }
    DbBatch dbBatch = new DbBatch(rowBatch);
    freedomExtractor.extract(dbBatch);
    want.collection(dbBatch.getRowBatch().getDatas()).sizeEq(count * count);
}
Also used : DataMediaPair(com.alibaba.otter.shared.common.model.config.data.DataMediaPair) RowBatch(com.alibaba.otter.shared.etl.model.RowBatch) EventData(com.alibaba.otter.shared.etl.model.EventData) DbBatch(com.alibaba.otter.shared.etl.model.DbBatch) Pipeline(com.alibaba.otter.shared.common.model.config.pipeline.Pipeline) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Example 42 with EventData

use of com.alibaba.otter.shared.etl.model.EventData in project otter by alibaba.

the class DbLoadActionTest method generatorEventDataForMysql.

private List<EventData> generatorEventDataForMysql(int start, int count, EventType type) {
    List<EventData> eventDatas = new ArrayList<EventData>();
    for (int i = 0; i < count; i++) {
        int index = i + 1 + start;
        EventData eventData = new EventData();
        eventData.setPairId(index);
        eventData.setTableId(1L);
        eventData.setSchemaName("srf");
        eventData.setTableName("columns");
        eventData.setEventType(type);
        eventData.setExecuteTime(100L);
        eventData.getKeys().add(buildColumn("id", Types.INTEGER, "" + index, true, false));
        eventData.getKeys().add(buildColumn("name", Types.VARCHAR, "ljh_" + index, true, false));
        eventData.getOldKeys().add(buildColumn("id", Types.INTEGER, "" + index, true, false));
        eventData.getOldKeys().add(buildColumn("name", Types.VARCHAR, "ljh_" + index, true, false));
        eventData.getColumns().add(buildColumn("alias_name", Types.CHAR, "hello_" + index, false, false));
        eventData.getColumns().add(buildColumn("amount", Types.DECIMAL, "100.01", false, false));
        eventData.getColumns().add(buildColumn("text_b", Types.BLOB, "[116,101,120,116,95,98]", false, false));
        eventData.getColumns().add(buildColumn("text_c", Types.CLOB, "中文", false, false));
        eventData.getColumns().add(buildColumn("curr_date", Types.DATE, "2011-01-01", false, false));
        eventData.getColumns().add(buildColumn("gmt_create", Types.TIMESTAMP, "2011-01-01 11:11:11", false, false));
        eventData.getColumns().add(buildColumn("gmt_modify", Types.TIMESTAMP, "2011-01-01 11:11:11", false, false));
        // MysqlSqlTemplate sqlTemplate = new MysqlSqlTemplate();
        // String sql = null;
        // if (type.isInsert()) {
        // sql = sqlTemplate.getMergeSql(eventData.getSchemaName(),
        // eventData.getTableName(),
        // buildColumnNames(eventData.getKeys()),
        // buildColumnNames(eventData.getColumns()), new String[] {});
        // } else if (type.isUpdate()) {
        // sql = sqlTemplate.getUpdateSql(eventData.getSchemaName(),
        // eventData.getTableName(),
        // buildColumnNames(eventData.getKeys()),
        // buildColumnNames(eventData.getColumns()));
        // } else if (type.isDelete()) {
        // sql = sqlTemplate.getDeleteSql(eventData.getSchemaName(),
        // eventData.getTableName(),
        // buildColumnNames(eventData.getKeys()));
        // }
        // eventData.setSql(sql);
        eventDatas.add(eventData);
    }
    return eventDatas;
}
Also used : ArrayList(java.util.ArrayList) EventData(com.alibaba.otter.shared.etl.model.EventData)

Example 43 with EventData

use of com.alibaba.otter.shared.etl.model.EventData in project otter by alibaba.

the class DbLoadActionTest method test_db_load_oracle.

@Test
public void test_db_load_oracle() {
    ArbitrateConfigRegistry.regist(configClientService);
    dbLoadAction = (DbLoadAction) TestedObject.getSpringBeanFactory().getBean("dbLoadAction");
    final Channel channel = new Channel();
    channel.setId(1L);
    final Pipeline pipeline = new Pipeline();
    pipeline.setId(100L);
    List<DataMediaPair> pairs = generatorDataMediaPairForOracle(20);
    pipeline.setPairs(pairs);
    pipeline.getParameters().merge(new SystemParameter());
    pipeline.getParameters().merge(new ChannelParameter());
    // final Pipeline oppositePipeline = new Pipeline();
    // oppositePipeline.setId(101L);
    channel.setPipelines(Arrays.asList(pipeline));
    final Node currentNode = new Node();
    currentNode.setId(1L);
    new NonStrictExpectations() {

        {
            configClientService.findChannel(anyLong);
            returns(channel);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
            configClientService.currentNode();
            returns(currentNode);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    List<EventData> eventDatas = generatorEventDataForOracle(0, 20, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForOracle(10, 10, EventType.INSERT);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    eventDatas = generatorEventDataForOracle(19, 1, EventType.DELETE);
    for (EventData eventData : eventDatas) {
        rowBatch.merge(eventData);
    }
    WeightController controller = new WeightController(1);
    dbLoadAction.load(rowBatch, controller);
}
Also used : DataMediaPair(com.alibaba.otter.shared.common.model.config.data.DataMediaPair) ChannelParameter(com.alibaba.otter.shared.common.model.config.channel.ChannelParameter) Channel(com.alibaba.otter.shared.common.model.config.channel.Channel) Node(com.alibaba.otter.shared.common.model.config.node.Node) EventData(com.alibaba.otter.shared.etl.model.EventData) Pipeline(com.alibaba.otter.shared.common.model.config.pipeline.Pipeline) RowBatch(com.alibaba.otter.shared.etl.model.RowBatch) SystemParameter(com.alibaba.otter.shared.common.model.config.parameter.SystemParameter) WeightController(com.alibaba.otter.node.etl.load.loader.weight.WeightController) Identity(com.alibaba.otter.shared.etl.model.Identity) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Example 44 with EventData

use of com.alibaba.otter.shared.etl.model.EventData in project otter by alibaba.

the class OtterTransformerFactory method transform.

/**
     * 将一种源数据进行转化,最后得到的结果会根据DataMediaPair中定义的目标对象生成不同的数据对象 <br/>
     * 
     * <pre>
     * 返回对象格式:Map
     * key : Class对象,代表生成的目标数据对象
     * value : 每种目标数据对象的集合数据
     * </pre>
     */
public Map<Class, BatchObject> transform(RowBatch rowBatch) {
    final Identity identity = translateIdentity(rowBatch.getIdentity());
    Map<Class, BatchObject> result = new HashMap<Class, BatchObject>();
    // 初始化默认值
    result.put(EventData.class, initBatchObject(identity, EventData.class));
    for (EventData eventData : rowBatch.getDatas()) {
        // 处理eventData
        Long tableId = eventData.getTableId();
        Pipeline pipeline = configClientService.findPipeline(identity.getPipelineId());
        // 针对每个同步数据,可能会存在多路复制的情况
        List<DataMediaPair> dataMediaPairs = ConfigHelper.findDataMediaPairByMediaId(pipeline, tableId);
        for (DataMediaPair pair : dataMediaPairs) {
            if (!pair.getSource().getId().equals(tableId)) {
                // 过滤tableID不为源的同步
                continue;
            }
            OtterTransformer translate = lookup(pair.getSource(), pair.getTarget());
            // 进行转化
            Object item = translate.transform(eventData, new OtterTransformerContext(identity, pair, pipeline));
            if (item == null) {
                continue;
            }
            // 合并结果
            merge(identity, result, item);
        }
    }
    return result;
}
Also used : DataMediaPair(com.alibaba.otter.shared.common.model.config.data.DataMediaPair) HashMap(java.util.HashMap) EventData(com.alibaba.otter.shared.etl.model.EventData) Pipeline(com.alibaba.otter.shared.common.model.config.pipeline.Pipeline) BatchObject(com.alibaba.otter.shared.etl.model.BatchObject) BatchObject(com.alibaba.otter.shared.etl.model.BatchObject) Identity(com.alibaba.otter.shared.etl.model.Identity)

Example 45 with EventData

use of com.alibaba.otter.shared.etl.model.EventData in project otter by alibaba.

the class DbLoadMergerTest method testMergeWithUpdateKeyOfIUUUU.

/**
     * 测试在主键发生变化后的merge操作,Insert/Update/Update/Update/Update
     */
@Test
public void testMergeWithUpdateKeyOfIUUUU() {
    Map<RowKey, EventData> mergeMap = new MapMaker().makeMap();
    DbLoadMerger.merge(makeInsertEventData(), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE, KEY_VALUE_NEW1), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE_NEW1, KEY_VALUE_NEW2), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE_NEW2, KEY_VALUE_NEW3), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE_NEW3, KEY_VALUE_NEW4), mergeMap);
    for (Entry<RowKey, EventData> entry : mergeMap.entrySet()) {
        RowKey key = entry.getKey();
        EventColumn keyColumn = key.getKeys().get(0);
        Assert.assertEquals(KEY_VALUE_NEW4, keyColumn.getColumnValue());
        Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());
        EventData eventData = entry.getValue();
        Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
        Assert.assertEquals(TABLE_NAME, eventData.getTableName());
        Assert.assertEquals(TABLE_ID, eventData.getTableId());
        Assert.assertEquals(EventType.INSERT, eventData.getEventType());
        // 不存在oldKeys
        Assert.assertEquals(eventData.getOldKeys().size(), 0);
    }
}
Also used : RowKey(com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey) EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) MapMaker(com.google.common.collect.MapMaker) EventData(com.alibaba.otter.shared.etl.model.EventData) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Aggregations

EventData (com.alibaba.otter.shared.etl.model.EventData)48 ArrayList (java.util.ArrayList)20 Pipeline (com.alibaba.otter.shared.common.model.config.pipeline.Pipeline)19 EventColumn (com.alibaba.otter.shared.etl.model.EventColumn)18 DataMediaPair (com.alibaba.otter.shared.common.model.config.data.DataMediaPair)16 Test (org.testng.annotations.Test)16 BaseDbTest (com.alibaba.otter.node.etl.BaseDbTest)15 RowBatch (com.alibaba.otter.shared.etl.model.RowBatch)14 Identity (com.alibaba.otter.shared.etl.model.Identity)9 MapMaker (com.google.common.collect.MapMaker)9 RowKey (com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey)8 DataMedia (com.alibaba.otter.shared.common.model.config.data.DataMedia)8 DbBatch (com.alibaba.otter.shared.etl.model.DbBatch)7 DbDialect (com.alibaba.otter.node.etl.common.db.dialect.DbDialect)5 ExtractException (com.alibaba.otter.node.etl.extract.exceptions.ExtractException)5 LoadException (com.alibaba.otter.node.etl.load.exception.LoadException)4 FileData (com.alibaba.otter.shared.etl.model.FileData)4 SelectException (com.alibaba.otter.node.etl.select.exceptions.SelectException)3 Channel (com.alibaba.otter.shared.common.model.config.channel.Channel)3 EventType (com.alibaba.otter.shared.etl.model.EventType)3