Search in sources :

Example 46 with EventData

use of com.alibaba.otter.shared.etl.model.EventData in project otter by alibaba.

the class DbLoadMergerTest method testMergeWithUpdateKeyOfUU.

/**
     * 测试在主键发生变化后的merge操作,Update/Update
     */
@Test
public void testMergeWithUpdateKeyOfUU() {
    Map<RowKey, EventData> mergeMap = new MapMaker().makeMap();
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE, KEY_VALUE_NEW1), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE_NEW1, KEY_VALUE_NEW2), mergeMap);
    for (Entry<RowKey, EventData> entry : mergeMap.entrySet()) {
        RowKey key = entry.getKey();
        EventColumn keyColumn = key.getKeys().get(0);
        Assert.assertEquals(KEY_VALUE_NEW2, keyColumn.getColumnValue());
        Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());
        EventData eventData = entry.getValue();
        Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
        Assert.assertEquals(TABLE_NAME, eventData.getTableName());
        Assert.assertEquals(TABLE_ID, eventData.getTableId());
        Assert.assertEquals(EventType.UPDATE, eventData.getEventType());
        List<EventColumn> oldKeys = eventData.getOldKeys();
        List<EventColumn> keys = eventData.getKeys();
        Assert.assertNotSame(oldKeys, keys);
    }
}
Also used : RowKey(com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey) EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) MapMaker(com.google.common.collect.MapMaker) EventData(com.alibaba.otter.shared.etl.model.EventData) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Example 47 with EventData

use of com.alibaba.otter.shared.etl.model.EventData in project otter by alibaba.

the class DbLoadMergerTest method testMergeWithSameKeyOfIUDI.

/**
     * 测试insert+update+delete+insert
     */
@Test
public void testMergeWithSameKeyOfIUDI() {
    Map<RowKey, EventData> mergeMap = new MapMaker().makeMap();
    DbLoadMerger.merge(makeInsertEventData(), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(), mergeMap);
    DbLoadMerger.merge(makeDeleteEventData(), mergeMap);
    DbLoadMerger.merge(makeInsertEventData(), mergeMap);
    for (Entry<RowKey, EventData> entry : mergeMap.entrySet()) {
        RowKey key = entry.getKey();
        EventColumn keyColumn = key.getKeys().get(0);
        Assert.assertEquals(KEY_VALUE, keyColumn.getColumnValue());
        Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());
        EventData eventData = entry.getValue();
        Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
        Assert.assertEquals(TABLE_NAME, eventData.getTableName());
        Assert.assertEquals(TABLE_ID, eventData.getTableId());
        Assert.assertEquals(EventType.INSERT, eventData.getEventType());
        // 不存在oldKeys
        Assert.assertEquals(eventData.getOldKeys().size(), 0);
        List<EventColumn> columns = eventData.getColumns();
        Assert.assertEquals(2, columns.size());
    }
}
Also used : RowKey(com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey) EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) MapMaker(com.google.common.collect.MapMaker) EventData(com.alibaba.otter.shared.etl.model.EventData) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Example 48 with EventData

use of com.alibaba.otter.shared.etl.model.EventData in project otter by alibaba.

the class DbLoadMergerTest method testMergeWithUpdateKeyOfII.

/**
     * 测试在主键发生变化后的merge操作,Insert/Insert
     */
@Test
public void testMergeWithUpdateKeyOfII() {
    Map<RowKey, EventData> mergeMap = new MapMaker().makeMap();
    DbLoadMerger.merge(makeInsertEventData(), mergeMap);
    DbLoadMerger.merge(makeInsertEventData(), mergeMap);
    for (Entry<RowKey, EventData> entry : mergeMap.entrySet()) {
        RowKey key = entry.getKey();
        EventColumn keyColumn = key.getKeys().get(0);
        Assert.assertEquals(KEY_VALUE, keyColumn.getColumnValue());
        Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());
        EventData eventData = entry.getValue();
        Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
        Assert.assertEquals(TABLE_NAME, eventData.getTableName());
        Assert.assertEquals(TABLE_ID, eventData.getTableId());
        Assert.assertEquals(EventType.INSERT, eventData.getEventType());
        List<EventColumn> oldKeys = eventData.getOldKeys();
        List<EventColumn> keys = eventData.getKeys();
        Assert.assertNotSame(oldKeys, keys);
    }
}
Also used : RowKey(com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey) EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) MapMaker(com.google.common.collect.MapMaker) EventData(com.alibaba.otter.shared.etl.model.EventData) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Aggregations

EventData (com.alibaba.otter.shared.etl.model.EventData)48 ArrayList (java.util.ArrayList)20 Pipeline (com.alibaba.otter.shared.common.model.config.pipeline.Pipeline)19 EventColumn (com.alibaba.otter.shared.etl.model.EventColumn)18 DataMediaPair (com.alibaba.otter.shared.common.model.config.data.DataMediaPair)16 Test (org.testng.annotations.Test)16 BaseDbTest (com.alibaba.otter.node.etl.BaseDbTest)15 RowBatch (com.alibaba.otter.shared.etl.model.RowBatch)14 Identity (com.alibaba.otter.shared.etl.model.Identity)9 MapMaker (com.google.common.collect.MapMaker)9 RowKey (com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey)8 DataMedia (com.alibaba.otter.shared.common.model.config.data.DataMedia)8 DbBatch (com.alibaba.otter.shared.etl.model.DbBatch)7 DbDialect (com.alibaba.otter.node.etl.common.db.dialect.DbDialect)5 ExtractException (com.alibaba.otter.node.etl.extract.exceptions.ExtractException)5 LoadException (com.alibaba.otter.node.etl.load.exception.LoadException)4 FileData (com.alibaba.otter.shared.etl.model.FileData)4 SelectException (com.alibaba.otter.node.etl.select.exceptions.SelectException)3 Channel (com.alibaba.otter.shared.common.model.config.channel.Channel)3 EventType (com.alibaba.otter.shared.etl.model.EventType)3