Search in sources :

Example 11 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class DbLoadMergerTest method makeUpdateEventData.

private EventData makeUpdateEventData() {
    EventData eventData = new EventData();
    eventData.setEventType(EventType.UPDATE);
    eventData.setSchemaName(SCHEMA_NAME);
    eventData.setTableName(TABLE_NAME);
    eventData.setTableId(TABLE_ID);
    List<EventColumn> keys = new ArrayList<EventColumn>();
    keys.add(makeEventColumn(KEY_NAME, KEY_VALUE, true));
    eventData.setKeys(keys);
    List<EventColumn> columns = new ArrayList<EventColumn>();
    int i = 0;
    for (String columnName : COLUMN_NAMES2) {
        columns.add(makeEventColumn(columnName, columnName + i, false));
        i++;
    }
    eventData.setColumns(columns);
    return eventData;
}
Also used : EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) ArrayList(java.util.ArrayList) EventData(com.alibaba.otter.shared.etl.model.EventData)

Example 12 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class DbLoadMergerTest method makeEventColumn.

private EventColumn makeEventColumn(String columnName, String columnValue, boolean key) {
    EventColumn eventColumn = new EventColumn();
    eventColumn.setColumnName(columnName);
    eventColumn.setColumnType(COLUMN_TYPE);
    eventColumn.setColumnValue(columnValue);
    eventColumn.setKey(key);
    return eventColumn;
}
Also used : EventColumn(com.alibaba.otter.shared.etl.model.EventColumn)

Example 13 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class DbLoadMergerTest method makeDeleteEventData.

private EventData makeDeleteEventData() {
    EventData eventData = new EventData();
    eventData.setEventType(EventType.DELETE);
    eventData.setSchemaName(SCHEMA_NAME);
    eventData.setTableName(TABLE_NAME);
    eventData.setTableId(TABLE_ID);
    List<EventColumn> keys = new ArrayList<EventColumn>();
    keys.add(makeEventColumn(KEY_NAME, KEY_VALUE, true));
    eventData.setKeys(keys);
    return eventData;
}
Also used : EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) ArrayList(java.util.ArrayList) EventData(com.alibaba.otter.shared.etl.model.EventData)

Example 14 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class DbLoadMergerTest method testMergeWithSameKeyOfIUD.

/**
     * 测试insert+update+delete
     */
@Test
public void testMergeWithSameKeyOfIUD() {
    Map<RowKey, EventData> mergeMap = new MapMaker().makeMap();
    DbLoadMerger.merge(makeInsertEventData(), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(), mergeMap);
    DbLoadMerger.merge(makeDeleteEventData(), mergeMap);
    for (Entry<RowKey, EventData> entry : mergeMap.entrySet()) {
        RowKey key = entry.getKey();
        EventColumn keyColumn = key.getKeys().get(0);
        Assert.assertEquals(KEY_VALUE, keyColumn.getColumnValue());
        Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());
        EventData eventData = entry.getValue();
        Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
        Assert.assertEquals(TABLE_NAME, eventData.getTableName());
        Assert.assertEquals(TABLE_ID, eventData.getTableId());
        Assert.assertEquals(EventType.DELETE, eventData.getEventType());
        // 不存在oldKeys
        Assert.assertEquals(eventData.getOldKeys().size(), 0);
        List<EventColumn> columns = eventData.getColumns();
        Assert.assertEquals(0, columns.size());
    }
}
Also used : RowKey(com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey) EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) MapMaker(com.google.common.collect.MapMaker) EventData(com.alibaba.otter.shared.etl.model.EventData) Test(org.testng.annotations.Test) BaseDbTest(com.alibaba.otter.node.etl.BaseDbTest)

Example 15 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class DbLoadMergerTest method makeUpdateEventData.

private EventData makeUpdateEventData(String oldKeyValue, String newKeyValue) {
    EventData eventData = new EventData();
    eventData.setEventType(EventType.UPDATE);
    eventData.setSchemaName(SCHEMA_NAME);
    eventData.setTableName(TABLE_NAME);
    eventData.setTableId(TABLE_ID);
    List<EventColumn> oldKeys = new ArrayList<EventColumn>();
    oldKeys.add(makeEventColumn(KEY_NAME, oldKeyValue, true));
    List<EventColumn> newKeys = new ArrayList<EventColumn>();
    newKeys.add(makeEventColumn(KEY_NAME, newKeyValue, true));
    eventData.setKeys(newKeys);
    eventData.setOldKeys(oldKeys);
    return eventData;
}
Also used : EventColumn(com.alibaba.otter.shared.etl.model.EventColumn) ArrayList(java.util.ArrayList) EventData(com.alibaba.otter.shared.etl.model.EventData)

Aggregations

EventColumn (com.alibaba.otter.shared.etl.model.EventColumn)38 EventData (com.alibaba.otter.shared.etl.model.EventData)18 ArrayList (java.util.ArrayList)13 Test (org.testng.annotations.Test)9 BaseDbTest (com.alibaba.otter.node.etl.BaseDbTest)8 RowKey (com.alibaba.otter.node.etl.load.loader.db.DbLoadMerger.RowKey)8 MapMaker (com.google.common.collect.MapMaker)8 DbDialect (com.alibaba.otter.node.etl.common.db.dialect.DbDialect)4 Pipeline (com.alibaba.otter.shared.common.model.config.pipeline.Pipeline)4 FileData (com.alibaba.otter.shared.etl.model.FileData)4 Table (org.apache.ddlutils.model.Table)4 ColumnPair (com.alibaba.otter.shared.common.model.config.data.ColumnPair)3 DataMedia (com.alibaba.otter.shared.common.model.config.data.DataMedia)3 EventType (com.alibaba.otter.shared.etl.model.EventType)3 FileBatch (com.alibaba.otter.shared.etl.model.FileBatch)3 RowBatch (com.alibaba.otter.shared.etl.model.RowBatch)3 File (java.io.File)3 PipeException (com.alibaba.otter.node.etl.common.pipe.exception.PipeException)2 BatchProto (com.alibaba.otter.node.etl.model.protobuf.BatchProto)2 TransformException (com.alibaba.otter.node.etl.transform.exception.TransformException)2