Examples with EventColumn com.alibaba.otter.shared.etl.model.EventColumn used on opensource projects

Example 26 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class RowDataHttpPipe method buildColumn.

private EventColumn buildColumn(BatchProto.Column columnProto) {
    EventColumn column = new EventColumn();
    column.setColumnName(columnProto.getName());
    column.setNull(columnProto.getIsNull());
    column.setColumnType(columnProto.getType());
    column.setColumnValue(columnProto.getValue());
    column.setKey(columnProto.getIsPrimaryKey());
    column.setIndex(columnProto.getIndex());
    // add by ljh
    column.setUpdate(columnProto.getIsUpdate());
    // 2012-08-30，标记变更字段
    return column;
}

Example 27 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class RowDataHttpPipe method getDbBatch.

// 处理对应的dbBatch
private DbBatch getDbBatch(HttpPipeKey key) {
    String dataUrl = key.getUrl();
    Pipeline pipeline = configClientService.findPipeline(key.getIdentity().getPipelineId());
    DataRetriever dataRetriever = dataRetrieverFactory.createRetriever(pipeline.getParameters().getRetriever(), dataUrl, downloadDir);
    File archiveFile = null;
    try {
        dataRetriever.connect();
        dataRetriever.doRetrieve();
        archiveFile = dataRetriever.getDataAsFile();
    } catch (Exception e) {
        dataRetriever.abort();
        throw new PipeException("download_error", e);
    } finally {
        dataRetriever.disconnect();
    }
    // 处理下有加密的数据
    if (StringUtils.isNotEmpty(key.getKey()) && StringUtils.isNotEmpty(key.getCrc())) {
        decodeFile(archiveFile, key.getKey(), key.getCrc());
    }
    InputStream input = null;
    JSONReader reader = null;
    try {
        input = new BufferedInputStream(new FileInputStream(archiveFile));
        DbBatch dbBatch = new DbBatch();
        byte[] lengthBytes = new byte[4];
        input.read(lengthBytes);
        int length = ByteUtils.bytes2int(lengthBytes);
        BatchProto.RowBatch rowbatchProto = BatchProto.RowBatch.parseFrom(new LimitedInputStream(input, length));
        // 构造原始的model对象
        RowBatch rowBatch = new RowBatch();
        rowBatch.setIdentity(build(rowbatchProto.getIdentity()));
        for (BatchProto.RowData rowDataProto : rowbatchProto.getRowsList()) {
            EventData eventData = new EventData();
            eventData.setPairId(rowDataProto.getPairId());
            eventData.setTableId(rowDataProto.getTableId());
            eventData.setTableName(rowDataProto.getTableName());
            eventData.setSchemaName(rowDataProto.getSchemaName());
            eventData.setEventType(EventType.valuesOf(rowDataProto.getEventType()));
            eventData.setExecuteTime(rowDataProto.getExecuteTime());
            // add by ljh at 2012-10-31
            if (StringUtils.isNotEmpty(rowDataProto.getSyncMode())) {
                eventData.setSyncMode(SyncMode.valuesOf(rowDataProto.getSyncMode()));
            }
            if (StringUtils.isNotEmpty(rowDataProto.getSyncConsistency())) {
                eventData.setSyncConsistency(SyncConsistency.valuesOf(rowDataProto.getSyncConsistency()));
            }
            // 处理主键
            List<EventColumn> keys = new ArrayList<EventColumn>();
            for (BatchProto.Column columnProto : rowDataProto.getKeysList()) {
                keys.add(buildColumn(columnProto));
            }
            eventData.setKeys(keys);
            // 处理old主键
            if (CollectionUtils.isEmpty(rowDataProto.getOldKeysList()) == false) {
                List<EventColumn> oldKeys = new ArrayList<EventColumn>();
                for (BatchProto.Column columnProto : rowDataProto.getOldKeysList()) {
                    oldKeys.add(buildColumn(columnProto));
                }
                eventData.setOldKeys(oldKeys);
            }
            // 处理具体的column value
            List<EventColumn> columns = new ArrayList<EventColumn>();
            for (BatchProto.Column columnProto : rowDataProto.getColumnsList()) {
                columns.add(buildColumn(columnProto));
            }
            eventData.setColumns(columns);
            eventData.setRemedy(rowDataProto.getRemedy());
            eventData.setSize(rowDataProto.getSize());
            eventData.setSql(rowDataProto.getSql());
            eventData.setDdlSchemaName(rowDataProto.getDdlSchemaName());
            eventData.setHint(rowDataProto.getHint());
            eventData.setWithoutSchema(rowDataProto.getWithoutSchema());
            // 添加到总记录
            rowBatch.merge(eventData);
        }
        dbBatch.setRowBatch(rowBatch);
        input.read(lengthBytes);
        length = ByteUtils.bytes2int(lengthBytes);
        BatchProto.FileBatch filebatchProto = BatchProto.FileBatch.parseFrom(new LimitedInputStream(input, length));
        // 构造原始的model对象
        FileBatch fileBatch = new FileBatch();
        fileBatch.setIdentity(build(filebatchProto.getIdentity()));
        for (BatchProto.FileData fileDataProto : filebatchProto.getFilesList()) {
            FileData fileData = new FileData();
            fileData.setPairId(fileDataProto.getPairId());
            fileData.setTableId(fileDataProto.getTableId());
            fileData.setEventType(EventType.valuesOf(fileDataProto.getEventType()));
            fileData.setLastModifiedTime(fileDataProto.getLastModifiedTime());
            fileData.setNameSpace(fileDataProto.getNamespace());
            fileData.setPath(fileDataProto.getPath());
            fileData.setSize(fileDataProto.getSize());
            // 添加到filebatch中
            fileBatch.getFiles().add(fileData);
        }
        dbBatch.setFileBatch(fileBatch);
        return dbBatch;
    } catch (IOException e) {
        throw new PipeException("deserial_error", e);
    } finally {
        IOUtils.closeQuietly(reader);
    }
}

Example 28 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class DbLoadDumper method dumpEventColumn.

private static String dumpEventColumn(List<EventColumn> columns) {
    StringBuilder builder = new StringBuilder(event_default_capacity);
    int size = columns.size();
    for (int i = 0; i < size; i++) {
        EventColumn column = columns.get(i);
        builder.append("\t").append(column.toString());
        if (i < columns.size() - 1) {
            builder.append(SEP);
        }
    }
    return builder.toString();
}

Example 29 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class MessageParser method internParse.

/**
     * 解析出从canal中获取的Event事件<br>
     * Oracle:有变更的列值. <br>
     * <i>insert:从afterColumns中获取所有的变更数据<br>
     * <i>delete:从beforeColumns中获取所有的变更数据<br>
     * <i>update:在before中存放所有的主键和变化前的非主键值，在after中存放变化后的主键和非主键值,如果是复合主键，只会存放变化的主键<br>
     * Mysql:可以得到所有变更前和变更后的数据.<br>
     * <i>insert:从afterColumns中获取所有的变更数据<br>
     * <i>delete:从beforeColumns中获取所有的变更数据<br>
     * <i>update:在beforeColumns中存放变更前的所有数据,在afterColumns中存放变更后的所有数据<br>
     */
private EventData internParse(Pipeline pipeline, Entry entry, RowChange rowChange, RowData rowData) {
    EventData eventData = new EventData();
    eventData.setTableName(entry.getHeader().getTableName());
    eventData.setSchemaName(entry.getHeader().getSchemaName());
    eventData.setEventType(EventType.valueOf(rowChange.getEventType().name()));
    eventData.setExecuteTime(entry.getHeader().getExecuteTime());
    EventType eventType = eventData.getEventType();
    TableInfoHolder tableHolder = null;
    if (!StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemSchema(), eventData.getSchemaName())) {
        boolean useTableTransform = pipeline.getParameters().getUseTableTransform();
        Table table = null;
        DataMediaPair dataMediaPair = ConfigHelper.findDataMediaPairBySourceName(pipeline, eventData.getSchemaName(), eventData.getTableName());
        DataMedia dataMedia = dataMediaPair.getSource();
        eventData.setTableId(dataMedia.getId());
        // 获取目标表
        DataMedia targetDataMedia = dataMediaPair.getTarget();
        if (useTableTransform || dataMedia.getSource().getType().isOracle()) {
            // oracle需要反查一次meta
            // 如果设置了需要进行table meta转化，则反查一下table信息
            // 比如oracle erosa解析时可能使用了非物理主键，需要直接使用，信任erosa的信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
            // 查询一下meta信息
            table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
            if (table == null) {
                logger.warn("find table[{}.{}] is null , may be drop table.", eventData.getSchemaName(), eventData.getTableName());
            }
            // 获取一下目标库的拆分字段,设置源表为主键
            // 首先要求源和目标的库名表名是一致的
            DbDialect targetDbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) targetDataMedia.getSource());
            if (targetDbDialect.isDRDS()) {
                String schemaName = buildName(eventData.getSchemaName(), dataMedia.getNamespaceMode(), targetDataMedia.getNamespaceMode());
                String tableName = buildName(eventData.getSchemaName(), dataMedia.getNameMode(), targetDataMedia.getNameMode());
                String shardColumns = targetDbDialect.getShardColumns(schemaName, tableName);
                if (StringUtils.isNotEmpty(shardColumns)) {
                    String[] columns = StringUtils.split(shardColumns, ',');
                    for (String key : columns) {
                        org.apache.ddlutils.model.Column col = table.findColumn(key, false);
                        if (col != null) {
                            col.setPrimaryKey(true);
                        } else {
                            logger.warn(String.format("shardColumn %s in table[%s.%s] is not found", key, eventData.getSchemaName(), eventData.getTableName()));
                        }
                    }
                }
            }
            tableHolder = new TableInfoHolder(dbDialect, table, useTableTransform);
        }
    }
    List<Column> beforeColumns = rowData.getBeforeColumnsList();
    List<Column> afterColumns = rowData.getAfterColumnsList();
    String tableName = eventData.getSchemaName() + "." + eventData.getTableName();
    // 判断一下是否需要all columns
    // 如果是rowMode模式，所有字段都需要标记为updated
    boolean isRowMode = pipeline.getParameters().getSyncMode().isRow();
    boolean needAllColumns = isRowMode || checkNeedAllColumns(pipeline);
    // 变更后的主键
    Map<String, EventColumn> keyColumns = new LinkedHashMap<String, EventColumn>();
    // 变更前的主键
    Map<String, EventColumn> oldKeyColumns = new LinkedHashMap<String, EventColumn>();
    // 有变化的非主键
    Map<String, EventColumn> notKeyColumns = new LinkedHashMap<String, EventColumn>();
    if (eventType.isInsert()) {
        for (Column column : afterColumns) {
            if (isKey(tableHolder, tableName, column)) {
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else {
                // mysql 有效
                notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            }
        }
    } else if (eventType.isDelete()) {
        for (Column column : beforeColumns) {
            if (isKey(tableHolder, tableName, column)) {
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else {
                // mysql 有效
                notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            }
        }
    } else if (eventType.isUpdate()) {
        // 获取变更前的主键.
        for (Column column : beforeColumns) {
            if (isKey(tableHolder, tableName, column)) {
                oldKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
                // 同时记录一下new
                // key,因为mysql5.6之后出现了minimal模式,after里会没有主键信息,需要在before记录中找
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else {
                if (needAllColumns && entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) {
                    // 针对行记录同步时，针对oracle记录一下非主键的字段，因为update时针对未变更的字段在aftercolume里没有
                    notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode, tableHolder));
                }
            }
        }
        for (Column column : afterColumns) {
            if (isKey(tableHolder, tableName, column)) {
                // 获取变更后的主键
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else if (needAllColumns || entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE || column.getUpdated()) {
                // 在update操作时，oracle和mysql存放变更的非主键值的方式不同,oracle只有变更的字段;
                // mysql会把变更前和变更后的字段都发出来，只需要取有变更的字段.
                // 如果是oracle库，after里一定为对应的变更字段
                boolean isUpdate = true;
                if (entry.getHeader().getSourceType() == CanalEntry.Type.MYSQL) {
                    // mysql的after里部分数据为未变更,oracle里after里为变更字段
                    isUpdate = column.getUpdated();
                }
                // 如果是rowMode，所有字段都为updated
                notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode || isUpdate, tableHolder));
            }
        }
        if (entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) {
            // 针对oracle进行特殊处理
            checkUpdateKeyColumns(oldKeyColumns, keyColumns);
        }
    }
    List<EventColumn> keys = new ArrayList<EventColumn>(keyColumns.values());
    List<EventColumn> oldKeys = new ArrayList<EventColumn>(oldKeyColumns.values());
    List<EventColumn> columns = new ArrayList<EventColumn>(notKeyColumns.values());
    Collections.sort(keys, new EventColumnIndexComparable());
    Collections.sort(oldKeys, new EventColumnIndexComparable());
    Collections.sort(columns, new EventColumnIndexComparable());
    if (!keyColumns.isEmpty()) {
        eventData.setKeys(keys);
        if (eventData.getEventType().isUpdate() && !oldKeys.equals(keys)) {
            // update类型，如果存在主键不同,则记录下old
            // keys为变更前的主键
            eventData.setOldKeys(oldKeys);
        }
        eventData.setColumns(columns);
    // } else if (CanalEntry.Type.MYSQL ==
    // entry.getHeader().getSourceType()) {
    // // 只支持mysql无主键同步
    // if (eventType.isUpdate()) {
    // List<EventColumn> oldColumns = new ArrayList<EventColumn>();
    // List<EventColumn> newColumns = new ArrayList<EventColumn>();
    // for (Column column : beforeColumns) {
    // oldColumns.add(copyEventColumn(column, true, tableHolder));
    // }
    //
    // for (Column column : afterColumns) {
    // newColumns.add(copyEventColumn(column, true, tableHolder));
    // }
    // Collections.sort(oldColumns, new EventColumnIndexComparable());
    // Collections.sort(newColumns, new EventColumnIndexComparable());
    // eventData.setOldKeys(oldColumns);// 做为老主键
    // eventData.setKeys(newColumns);// 做为新主键，需要保证新老主键字段数量一致
    // } else {
    // // 针对无主键，等同为所有都是主键进行处理
    // eventData.setKeys(columns);
    // }
    } else {
        throw new SelectException("this rowdata has no pks , entry: " + entry.toString() + " and rowData: " + rowData);
    }
    return eventData;
}

Example 30 with EventColumn

use of com.alibaba.otter.shared.etl.model.EventColumn in project otter by alibaba.

the class OtterTransformerTest method buildColumn.

private EventColumn buildColumn(String name, int type, String value, boolean isKey, boolean isNull) {
    EventColumn column = new EventColumn();
    column.setColumnName(name);
    column.setColumnType(type);
    column.setColumnValue(value);
    column.setKey(isKey);
    column.setNull(isNull);
    return column;
}