Examples with EventColumn com.alibaba.otter.shared.etl.model.EventColumn used on opensource projects

Example 31 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class TestEventProcessor method process.

public boolean process(EventData eventData) {
    // 基本步骤：
    // 1. 获取binlog中的变更字段
    // 2. 根据业务逻辑进行判断，如果需要忽略本条数据同步，直接返回false，否则返回true
    // 3. 根据业务逻辑进行逻辑转化，比如可以修改整个EventData数据.  
    // 本文例子：源库的每条binlog变更，记录到一个日志表binlog
    // create table test.binlog(
    //        id bigint(20) auto_increment,
    //        oschema varchar(256),
    //        otable varchar(256),
    //        gtime varchar(32)
    //        ovalue text,
    //        primary key(id);
    //    )
    // 在process处理中，可以修改EventData的任何数据，达到数据转换的效果, just have fun.
    JSONObject col = new JSONObject();
    JSONArray array = new JSONArray();
    for (EventColumn column : eventData.getColumns()) {
        JSONObject obj = this.doColumn(column);
        array.add(obj);
    }
    for (EventColumn key : eventData.getKeys()) {
        JSONObject obj = this.doColumn(key);
        array.add(obj);
    }
    col.put("schema", eventData.getSchemaName());
    col.put("table", eventData.getTableName());
    col.put("columns", array);
    col.put("dml", eventData.getEventType());
    col.put("exectime", eventData.getExecuteTime());
    // 构造新的主键
    EventColumn id = new EventColumn();
    id.setColumnValue(eventData.getSchemaName());
    id.setColumnType(Types.BIGINT);
    id.setColumnName("id");
    // 构造新的字段
    EventColumn schema = new EventColumn();
    schema.setColumnValue(eventData.getSchemaName());
    schema.setColumnType(Types.VARCHAR);
    schema.setColumnName("oschema");
    EventColumn table = new EventColumn();
    table.setColumnValue(eventData.getTableName());
    table.setColumnType(Types.VARCHAR);
    table.setColumnName("otable");
    EventColumn ovalue = new EventColumn();
    ovalue.setColumnValue(col.toJSONString());
    ovalue.setColumnType(Types.VARCHAR);
    ovalue.setColumnName("ovalue");
    EventColumn gtime = new EventColumn();
    gtime.setColumnValue(eventData.getExecuteTime() + "");
    gtime.setColumnType(Types.VARCHAR);
    gtime.setColumnName("gtime");
    // 替换为新的字段和主键信息
    List<EventColumn> cols = new ArrayList<EventColumn>();
    cols.add(schema);
    cols.add(table);
    cols.add(gtime);
    cols.add(ovalue);
    eventData.setColumns(cols);
    List<EventColumn> keys = new ArrayList<EventColumn>();
    keys.add(id);
    eventData.setKeys(keys);
    //修改数据meta信息
    eventData.setEventType(EventType.INSERT);
    eventData.setSchemaName("test");
    eventData.setTableName("binlog");
    return true;
}

Example 32 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class RowDataTransformer method translateColumn.

private EventColumn translateColumn(EventData data, EventColumn scolumn, TableInfoHolder tableHolder, DataMediaPair dataMediaPair, Multimap<String, String> translateColumnNames) {
    EventType type = data.getEventType();
    EventColumn tcolumn = new EventColumn();
    tcolumn.setNull(scolumn.getColumnValue() == null);
    // 左右两边的主键值必须保持一样，可以不为物理主键
    tcolumn.setKey(scolumn.isKey());
    tcolumn.setIndex(scolumn.getIndex());
    tcolumn.setUpdate(scolumn.isUpdate());
    String columnName = translateColumnName(scolumn.getColumnName(), dataMediaPair, translateColumnNames);
    if (StringUtils.isBlank(columnName)) {
        throw new TransformException("can't translate column name:" + scolumn.getColumnName() + "in pair:" + dataMediaPair.toString());
    }
    // 特殊处理
    // columnName = StringUtils.remove(columnName, "`"); //
    // 处理下特殊字符，eromanga给了错误的字段名
    tcolumn.setColumnName(columnName);
    // 不反查，直接使用源库的类型
    tcolumn.setColumnType(scolumn.getColumnType());
    if (tableHolder != null) {
        // modify by ljh at 2013-01-23
        // 双向同步新增字段，在一边加了字段后，虽然新的字段没有产生业务变化，但会因为某些原因导致传递了新的字段到T模块
        // 此时在目标库并不存在这个字段，导致一直挂起。ps. mysql新增字段时间不是一般的长
        // 所以，做了一个容错处理，针对目标库不存在的字段，如果变更记录在源库不存在变更，并且是null值的，允许丢弃该字段(其实最好还是要判断源库的column的defaultValue和当前值是否一致)
        boolean canColumnsNotExist = tableHolder.isEnableCompatibleMissColumn();
        if (type == EventType.UPDATE) {
            // 非变更字段，且当前值为null
            canColumnsNotExist &= !scolumn.isUpdate() && scolumn.isNull();
        } else if (type == EventType.INSERT) {
            // 当前值为null
            canColumnsNotExist &= scolumn.isNull();
        } else if (type == EventType.DELETE) {
            // 主键不允许不存在
            canColumnsNotExist &= !scolumn.isKey();
        }
        Column matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
        // 匹配字段为空，可能源库发生过DDL操作，目标库重新载入一下meta信息
        if (matchDbColumn == null) {
            // 尝试reload一下table meta
            // 获取目标库的表信息
            DbMediaSource dbMediaSource = (DbMediaSource) dataMediaPair.getTarget().getSource();
            DbDialect dbDialect = dbDialectFactory.getDbDialect(dataMediaPair.getPipelineId(), dbMediaSource);
            String schemaName = tableHolder.getTable().getSchema();
            if (StringUtils.isEmpty(schemaName)) {
                schemaName = tableHolder.getTable().getCatalog();
            }
            // 强制反查一次，并放入cache
            Table table = dbDialect.findTable(schemaName, tableHolder.getTable().getName(), false);
            tableHolder.setTable(table);
            matchDbColumn = getMatchColumn(tableHolder.getTable().getColumns(), tcolumn.getColumnName());
            if (matchDbColumn == null) {
                if (canColumnsNotExist) {
                    return null;
                } else {
                    throw new TransformException(scolumn.getColumnName() + " is not found in " + table.toString() + " and source : " + dataMediaPair.getTarget().getNamespace() + "." + dataMediaPair.getTarget().getName());
                }
            }
        }
        if (tableHolder.isUseTableTransform()) {
            int sqlType = matchDbColumn.getTypeCode();
            tcolumn.setColumnType(sqlType);
        }
    }
    // if (dataMediaPair.getTarget().getSource().getType().isOracle()) {
    // // 特殊处理下oracle编码
    // String encodeValue = SqlUtils.encoding(scolumn.getColumnValue(),
    // scolumn.getColumnType(),
    // dataMediaPair.getSource().getSource().getEncode(),
    // dataMediaPair.getTarget().getSource().getEncode());
    // tcolumn.setColumnValue(encodeValue);
    // } else {
    // mysql编码转化已经在驱动层面上完成
    tcolumn.setColumnValue(scolumn.getColumnValue());
    // }
    // 删除映射关系，避免下次重复转换
    translateColumnNames.remove(scolumn.getColumnName(), columnName);
    return tcolumn;
}

Example 33 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class HttpPipeIntegration method test_rowData.

@Test
public void test_rowData() {
    final Node currentNode = new Node();
    currentNode.setId(1L);
    currentNode.setIp("127.0.0.1");
    currentNode.setParameters(new NodeParameter());
    final Pipeline pipeline = new Pipeline();
    pipeline.getParameters().setRetriever(RetrieverType.ARIA2C);
    // mock一下
    new NonStrictExpectations() {

        {
            configClientService.currentNode();
            returns(currentNode);
            configClientService.findPipeline(anyLong);
            returns(pipeline);
        }
    };
    Identity identity = new Identity();
    identity.setChannelId(100L);
    identity.setPipelineId(100L);
    identity.setProcessId(100L);
    FileBatch fileBatch = new FileBatch();
    fileBatch.setIdentity(identity);
    File localFile = new File(tmp, "httpPipeTest.jpg");
    FileData localFileData = new FileData();
    localFileData.setPath(localFile.getPath());
    localFileData.setEventType(EventType.INSERT);
    localFileData.setLastModifiedTime(new Date().getTime());
    localFileData.setSize(100L);
    localFileData.setTableId(1L);
    fileBatch.getFiles().add(localFileData);
    RowBatch rowBatch = new RowBatch();
    rowBatch.setIdentity(identity);
    EventData eventData = new EventData();
    eventData.setTableId(1L);
    eventData.setSchemaName("otter");
    eventData.setTableName("test");
    eventData.setEventType(EventType.INSERT);
    eventData.setExecuteTime(100L);
    EventColumn primaryKey = new EventColumn();
    primaryKey.setColumnName("id");
    primaryKey.setColumnType(1);
    primaryKey.setColumnValue("1");
    primaryKey.setKey(true);
    primaryKey.setNull(false);
    eventData.getKeys().add(primaryKey);
    EventColumn column = new EventColumn();
    column.setColumnName("name");
    column.setColumnType(1);
    column.setColumnValue("test");
    column.setKey(false);
    column.setNull(false);
    eventData.getColumns().add(column);
    rowBatch.merge(eventData);
    DbBatch dbBatch = new DbBatch();
    dbBatch.setRowBatch(rowBatch);
    dbBatch.setFileBatch(fileBatch);
    HttpPipeKey key = rowDataHttpPipe.put(dbBatch);
    DbBatch target = rowDataHttpPipe.get(key);
    want.bool(target.getRowBatch().getIdentity().equals(identity));
    want.object(target).notNull();
}

Example 34 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class FreedomExtractorTest method getColumn.

private List<EventColumn> getColumn(int value) {
    EventColumn c1 = new EventColumn();
    c1.setColumnName("type");
    c1.setColumnValue("I");
    EventColumn c2 = new EventColumn();
    c2.setColumnName("table_id");
    c2.setColumnValue(String.valueOf(value));
    EventColumn c3 = new EventColumn();
    c3.setColumnName("pk_data");
    c3.setColumnValue(String.valueOf(value) + ((char) 1) + "hello");
    return Arrays.asList(c1, c2, c3);
}

Example 35 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class DbLoadMergerTest method testMergeWithUpdateKeyOfIUUUU.

/**
     * 测试在主键发生变化后的merge操作，Insert/Update/Update/Update/Update
     */
@Test
public void testMergeWithUpdateKeyOfIUUUU() {
    Map<RowKey, EventData> mergeMap = new MapMaker().makeMap();
    DbLoadMerger.merge(makeInsertEventData(), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE, KEY_VALUE_NEW1), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE_NEW1, KEY_VALUE_NEW2), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE_NEW2, KEY_VALUE_NEW3), mergeMap);
    DbLoadMerger.merge(makeUpdateEventData(KEY_VALUE_NEW3, KEY_VALUE_NEW4), mergeMap);
    for (Entry<RowKey, EventData> entry : mergeMap.entrySet()) {
        RowKey key = entry.getKey();
        EventColumn keyColumn = key.getKeys().get(0);
        Assert.assertEquals(KEY_VALUE_NEW4, keyColumn.getColumnValue());
        Assert.assertEquals(KEY_NAME, keyColumn.getColumnName());
        EventData eventData = entry.getValue();
        Assert.assertEquals(SCHEMA_NAME, eventData.getSchemaName());
        Assert.assertEquals(TABLE_NAME, eventData.getTableName());
        Assert.assertEquals(TABLE_ID, eventData.getTableId());
        Assert.assertEquals(EventType.INSERT, eventData.getEventType());
        // 不存在oldKeys
        Assert.assertEquals(eventData.getOldKeys().size(), 0);
    }
}