Examples with Column com.alibaba.otter.canal.protocol.CanalEntry.Column used on opensource projects

Example 1 with Column

use of com.alibaba.otter.canal.protocol.CanalEntry.Column in project canal by alibaba.

the class AbstractCanalClientTest method printColumn.

protected void printColumn(List<Column> columns) {
    for (Column column : columns) {
        StringBuilder builder = new StringBuilder();
        builder.append(column.getName() + " : " + column.getValue());
        builder.append("    type=" + column.getMysqlType());
        if (column.getUpdated()) {
            builder.append("    update=" + column.getUpdated());
        }
        builder.append(SEP);
        logger.info(builder.toString());
    }
}

Example 2 with Column

use of com.alibaba.otter.canal.protocol.CanalEntry.Column in project otter by alibaba.

the class MessageParser method checkLoopback.

/**
     * <pre>
     * the table def: 
     *              channel_info varchar
     *              channel_id varchar
     * 每次解析时，每个事务首先获取 retl_mark 下的 channel_info 或 channel_id 字段变更。
     *  a. 如果存在 channel_info 以 '_SYNC'结尾的字符串 ，则忽略本次事务的数据变更；
     *  b. 如果不等于，则执行下面的判断。
     *      i. 如果存在channel_id = "xx"，则检查对应的channel_id是否为当前同步的channelId，如果是则忽略。
     *      ii. 不存在则不处理
     * </pre>
     */
private int checkLoopback(Pipeline pipeline, RowData rowData) {
    // 检查channel_info字段
    // 首先检查下after记录，从无变有的过程，一般出现在事务头
    Column infokColumn = getColumnIgnoreCase(rowData.getAfterColumnsList(), pipeline.getParameters().getSystemMarkTableInfo());
    // 匹配对应的channelInfo，如果以_SYNC结尾，则认为需要忽略
    if (infokColumn != null && StringUtils.endsWithIgnoreCase(infokColumn.getValue(), RETL_CLIENT_FLAG)) {
        return 1;
    }
    // 匹配对应的channelInfo，如果相同，则认为需要忽略，并返回2，代表需要进行回环补救check机制，因为这个变更也是otter系统产生的
    if (infokColumn != null && StringUtils.equalsIgnoreCase(infokColumn.getValue(), pipeline.getParameters().getChannelInfo())) {
        return 2;
    }
    infokColumn = getColumnIgnoreCase(rowData.getBeforeColumnsList(), pipeline.getParameters().getSystemMarkTableInfo());
    // 匹配对应的channelInfo，如果以_SYNC结尾，则认为需要忽略
    if (infokColumn != null && StringUtils.endsWithIgnoreCase(infokColumn.getValue(), RETL_CLIENT_FLAG)) {
        return 1;
    }
    // 匹配对应的channelInfo，如果相同，则认为需要忽略，并返回2，代表需要进行回环补救check机制，因为这个变更也是otter系统产生的
    if (infokColumn != null && StringUtils.equalsIgnoreCase(infokColumn.getValue(), pipeline.getParameters().getChannelInfo())) {
        return 2;
    }
    // 检查channel_id字段
    Column markColumn = getColumnIgnoreCase(rowData.getAfterColumnsList(), pipeline.getParameters().getSystemMarkTableColumn());
    // 匹配对应的channel id
    if (markColumn != null && pipeline.getChannelId().equals(Long.parseLong(markColumn.getValue()))) {
        return 2;
    }
    markColumn = getColumnIgnoreCase(rowData.getBeforeColumnsList(), pipeline.getParameters().getSystemMarkTableColumn());
    if (markColumn != null && pipeline.getChannelId().equals(Long.parseLong(markColumn.getValue()))) {
        return 2;
    }
    return 0;
}

Example 3 with Column

use of com.alibaba.otter.canal.protocol.CanalEntry.Column in project canal by alibaba.

the class LogEventConvert method parseOneRow.

private boolean parseOneRow(RowData.Builder rowDataBuilder, RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException {
    int columnCnt = event.getTable().getColumnCnt();
    ColumnInfo[] columnInfo = event.getTable().getColumnInfo();
    boolean tableError = false;
    // check table fileds count，只能处理加字段
    boolean existRDSNoPrimaryKey = false;
    if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
        if (tableMetaCache.isOnRDS()) {
            // 特殊处理下RDS的场景
            List<FieldMeta> primaryKeys = tableMeta.getPrimaryFields();
            if (primaryKeys == null || primaryKeys.isEmpty()) {
                if (columnInfo.length == tableMeta.getFileds().size() + 1 && columnInfo[columnInfo.length - 1].type == LogEvent.MYSQL_TYPE_LONGLONG) {
                    existRDSNoPrimaryKey = true;
                }
            }
        }
        if (!existRDSNoPrimaryKey) {
            // online ddl增加字段操作步骤：
            // 1. 新增一张临时表，将需要做ddl表的数据全量导入
            // 2. 在老表上建立I/U/D的trigger，增量的将数据插入到临时表
            // 3. 锁住应用请求，将临时表rename为老表的名字，完成增加字段的操作
            // 尝试做一次reload，可能因为ddl没有正确解析，或者使用了类似online ddl的操作
            // 因为online ddl没有对应表名的alter语法，所以不会有clear cache的操作
            // 强制重新获取一次
            tableMeta = getTableMeta(event.getTable().getDbName(), event.getTable().getTableName(), false);
            if (tableMeta == null) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("not found [" + event.getTable().getDbName() + "." + event.getTable().getTableName() + "] in db , pls check!");
                }
            }
            // 在做一次判断
            if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("column size is not match for table:" + tableMeta.getFullName() + "," + columnInfo.length + " vs " + tableMeta.getFileds().size());
                }
            }
        } else {
            logger.warn("[" + event.getTable().getDbName() + "." + event.getTable().getTableName() + "] is no primary key , skip alibaba_rds_row_id column");
        }
    }
    for (int i = 0; i < columnCnt; i++) {
        ColumnInfo info = columnInfo[i];
        // mysql 5.6开始支持nolob/mininal类型,并不一定记录所有的列,需要进行判断
        if (!cols.get(i)) {
            continue;
        }
        if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
            // 不解析最后一列
            buffer.nextValue(info.type, info.meta, false);
            continue;
        }
        Column.Builder columnBuilder = Column.newBuilder();
        FieldMeta fieldMeta = null;
        if (tableMeta != null && !tableError) {
            // 处理file meta
            fieldMeta = tableMeta.getFileds().get(i);
            columnBuilder.setName(fieldMeta.getColumnName());
            columnBuilder.setIsKey(fieldMeta.isKey());
            // 增加mysql type类型,issue 73
            columnBuilder.setMysqlType(fieldMeta.getColumnType());
        }
        columnBuilder.setIndex(i);
        columnBuilder.setIsNull(false);
        // fixed issue
        // https://github.com/alibaba/canal/issues/66，特殊处理binary/varbinary，不能做编码处理
        boolean isBinary = false;
        if (fieldMeta != null) {
            if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "VARBINARY")) {
                isBinary = true;
            } else if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "BINARY")) {
                isBinary = true;
            }
        }
        buffer.nextValue(info.type, info.meta, isBinary);
        if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
            // 不解析最后一列
            continue;
        }
        int javaType = buffer.getJavaType();
        if (buffer.isNull()) {
            columnBuilder.setIsNull(true);
        } else {
            final Serializable value = buffer.getValue();
            // 处理各种类型
            switch(javaType) {
                case Types.INTEGER:
                case Types.TINYINT:
                case Types.SMALLINT:
                case Types.BIGINT:
                    // 处理unsigned类型
                    Number number = (Number) value;
                    if (fieldMeta != null && fieldMeta.isUnsigned() && number.longValue() < 0) {
                        switch(buffer.getLength()) {
                            case 1:
                                /* MYSQL_TYPE_TINY */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.SMALLINT;
                                break;
                            case 2:
                                /* MYSQL_TYPE_SHORT */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(SMALLINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.INTEGER;
                                break;
                            case 3:
                                /* MYSQL_TYPE_INT24 */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(MEDIUMINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.INTEGER;
                                break;
                            case 4:
                                /* MYSQL_TYPE_LONG */
                                columnBuilder.setValue(String.valueOf(Long.valueOf(INTEGER_MAX_VALUE + number.longValue())));
                                // 往上加一个量级
                                javaType = Types.BIGINT;
                                break;
                            case 8:
                                /* MYSQL_TYPE_LONGLONG */
                                columnBuilder.setValue(BIGINT_MAX_VALUE.add(BigInteger.valueOf(number.longValue())).toString());
                                // 往上加一个量级，避免执行出错
                                javaType = Types.DECIMAL;
                                break;
                        }
                    } else {
                        // 对象为number类型，直接valueof即可
                        columnBuilder.setValue(String.valueOf(value));
                    }
                    break;
                // float
                case Types.REAL:
                case // double
                Types.DOUBLE:
                    // 对象为number类型，直接valueof即可
                    columnBuilder.setValue(String.valueOf(value));
                    break;
                case // bit
                Types.BIT:
                    // 对象为number类型
                    columnBuilder.setValue(String.valueOf(value));
                    break;
                case Types.DECIMAL:
                    columnBuilder.setValue(((BigDecimal) value).toPlainString());
                    break;
                case Types.TIMESTAMP:
                // break;
                case Types.TIME:
                case Types.DATE:
                    // 需要处理year
                    columnBuilder.setValue(value.toString());
                    break;
                case Types.BINARY:
                case Types.VARBINARY:
                case Types.LONGVARBINARY:
                    // meta，按编码解析text
                    if (fieldMeta != null && isText(fieldMeta.getColumnType())) {
                        columnBuilder.setValue(new String((byte[]) value, charset));
                        javaType = Types.CLOB;
                    } else {
                        // byte数组，直接使用iso-8859-1保留对应编码，浪费内存
                        columnBuilder.setValue(new String((byte[]) value, ISO_8859_1));
                        javaType = Types.BLOB;
                    }
                    break;
                case Types.CHAR:
                case Types.VARCHAR:
                    columnBuilder.setValue(value.toString());
                    break;
                default:
                    columnBuilder.setValue(value.toString());
            }
        }
        columnBuilder.setSqlType(javaType);
        // 设置是否update的标记位
        columnBuilder.setUpdated(isAfter && isUpdate(rowDataBuilder.getBeforeColumnsList(), columnBuilder.getIsNull() ? null : columnBuilder.getValue(), i));
        if (isAfter) {
            rowDataBuilder.addAfterColumns(columnBuilder.build());
        } else {
            rowDataBuilder.addBeforeColumns(columnBuilder.build());
        }
    }
    return tableError;
}

Example 4 with Column

use of com.alibaba.otter.canal.protocol.CanalEntry.Column in project otter by alibaba.

the class MessageParser method internParse.

/**
     * 解析出从canal中获取的Event事件<br>
     * Oracle:有变更的列值. <br>
     * <i>insert:从afterColumns中获取所有的变更数据<br>
     * <i>delete:从beforeColumns中获取所有的变更数据<br>
     * <i>update:在before中存放所有的主键和变化前的非主键值，在after中存放变化后的主键和非主键值,如果是复合主键，只会存放变化的主键<br>
     * Mysql:可以得到所有变更前和变更后的数据.<br>
     * <i>insert:从afterColumns中获取所有的变更数据<br>
     * <i>delete:从beforeColumns中获取所有的变更数据<br>
     * <i>update:在beforeColumns中存放变更前的所有数据,在afterColumns中存放变更后的所有数据<br>
     */
private EventData internParse(Pipeline pipeline, Entry entry, RowChange rowChange, RowData rowData) {
    EventData eventData = new EventData();
    eventData.setTableName(entry.getHeader().getTableName());
    eventData.setSchemaName(entry.getHeader().getSchemaName());
    eventData.setEventType(EventType.valueOf(rowChange.getEventType().name()));
    eventData.setExecuteTime(entry.getHeader().getExecuteTime());
    EventType eventType = eventData.getEventType();
    TableInfoHolder tableHolder = null;
    if (!StringUtils.equalsIgnoreCase(pipeline.getParameters().getSystemSchema(), eventData.getSchemaName())) {
        boolean useTableTransform = pipeline.getParameters().getUseTableTransform();
        Table table = null;
        DataMediaPair dataMediaPair = ConfigHelper.findDataMediaPairBySourceName(pipeline, eventData.getSchemaName(), eventData.getTableName());
        DataMedia dataMedia = dataMediaPair.getSource();
        eventData.setTableId(dataMedia.getId());
        // 获取目标表
        DataMedia targetDataMedia = dataMediaPair.getTarget();
        if (useTableTransform || dataMedia.getSource().getType().isOracle()) {
            // oracle需要反查一次meta
            // 如果设置了需要进行table meta转化，则反查一下table信息
            // 比如oracle erosa解析时可能使用了非物理主键，需要直接使用，信任erosa的信息
            DbDialect dbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) dataMedia.getSource());
            // 查询一下meta信息
            table = dbDialect.findTable(eventData.getSchemaName(), eventData.getTableName());
            if (table == null) {
                logger.warn("find table[{}.{}] is null , may be drop table.", eventData.getSchemaName(), eventData.getTableName());
            }
            // 获取一下目标库的拆分字段,设置源表为主键
            // 首先要求源和目标的库名表名是一致的
            DbDialect targetDbDialect = dbDialectFactory.getDbDialect(pipeline.getId(), (DbMediaSource) targetDataMedia.getSource());
            if (targetDbDialect.isDRDS()) {
                String schemaName = buildName(eventData.getSchemaName(), dataMedia.getNamespaceMode(), targetDataMedia.getNamespaceMode());
                String tableName = buildName(eventData.getSchemaName(), dataMedia.getNameMode(), targetDataMedia.getNameMode());
                String shardColumns = targetDbDialect.getShardColumns(schemaName, tableName);
                if (StringUtils.isNotEmpty(shardColumns)) {
                    String[] columns = StringUtils.split(shardColumns, ',');
                    for (String key : columns) {
                        org.apache.ddlutils.model.Column col = table.findColumn(key, false);
                        if (col != null) {
                            col.setPrimaryKey(true);
                        } else {
                            logger.warn(String.format("shardColumn %s in table[%s.%s] is not found", key, eventData.getSchemaName(), eventData.getTableName()));
                        }
                    }
                }
            }
            tableHolder = new TableInfoHolder(dbDialect, table, useTableTransform);
        }
    }
    List<Column> beforeColumns = rowData.getBeforeColumnsList();
    List<Column> afterColumns = rowData.getAfterColumnsList();
    String tableName = eventData.getSchemaName() + "." + eventData.getTableName();
    // 判断一下是否需要all columns
    // 如果是rowMode模式，所有字段都需要标记为updated
    boolean isRowMode = pipeline.getParameters().getSyncMode().isRow();
    boolean needAllColumns = isRowMode || checkNeedAllColumns(pipeline);
    // 变更后的主键
    Map<String, EventColumn> keyColumns = new LinkedHashMap<String, EventColumn>();
    // 变更前的主键
    Map<String, EventColumn> oldKeyColumns = new LinkedHashMap<String, EventColumn>();
    // 有变化的非主键
    Map<String, EventColumn> notKeyColumns = new LinkedHashMap<String, EventColumn>();
    if (eventType.isInsert()) {
        for (Column column : afterColumns) {
            if (isKey(tableHolder, tableName, column)) {
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else {
                // mysql 有效
                notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            }
        }
    } else if (eventType.isDelete()) {
        for (Column column : beforeColumns) {
            if (isKey(tableHolder, tableName, column)) {
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else {
                // mysql 有效
                notKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            }
        }
    } else if (eventType.isUpdate()) {
        // 获取变更前的主键.
        for (Column column : beforeColumns) {
            if (isKey(tableHolder, tableName, column)) {
                oldKeyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
                // 同时记录一下new
                // key,因为mysql5.6之后出现了minimal模式,after里会没有主键信息,需要在before记录中找
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else {
                if (needAllColumns && entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) {
                    // 针对行记录同步时，针对oracle记录一下非主键的字段，因为update时针对未变更的字段在aftercolume里没有
                    notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode, tableHolder));
                }
            }
        }
        for (Column column : afterColumns) {
            if (isKey(tableHolder, tableName, column)) {
                // 获取变更后的主键
                keyColumns.put(column.getName(), copyEventColumn(column, true, tableHolder));
            } else if (needAllColumns || entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE || column.getUpdated()) {
                // 在update操作时，oracle和mysql存放变更的非主键值的方式不同,oracle只有变更的字段;
                // mysql会把变更前和变更后的字段都发出来，只需要取有变更的字段.
                // 如果是oracle库，after里一定为对应的变更字段
                boolean isUpdate = true;
                if (entry.getHeader().getSourceType() == CanalEntry.Type.MYSQL) {
                    // mysql的after里部分数据为未变更,oracle里after里为变更字段
                    isUpdate = column.getUpdated();
                }
                // 如果是rowMode，所有字段都为updated
                notKeyColumns.put(column.getName(), copyEventColumn(column, isRowMode || isUpdate, tableHolder));
            }
        }
        if (entry.getHeader().getSourceType() == CanalEntry.Type.ORACLE) {
            // 针对oracle进行特殊处理
            checkUpdateKeyColumns(oldKeyColumns, keyColumns);
        }
    }
    List<EventColumn> keys = new ArrayList<EventColumn>(keyColumns.values());
    List<EventColumn> oldKeys = new ArrayList<EventColumn>(oldKeyColumns.values());
    List<EventColumn> columns = new ArrayList<EventColumn>(notKeyColumns.values());
    Collections.sort(keys, new EventColumnIndexComparable());
    Collections.sort(oldKeys, new EventColumnIndexComparable());
    Collections.sort(columns, new EventColumnIndexComparable());
    if (!keyColumns.isEmpty()) {
        eventData.setKeys(keys);
        if (eventData.getEventType().isUpdate() && !oldKeys.equals(keys)) {
            // update类型，如果存在主键不同,则记录下old
            // keys为变更前的主键
            eventData.setOldKeys(oldKeys);
        }
        eventData.setColumns(columns);
    // } else if (CanalEntry.Type.MYSQL ==
    // entry.getHeader().getSourceType()) {
    // // 只支持mysql无主键同步
    // if (eventType.isUpdate()) {
    // List<EventColumn> oldColumns = new ArrayList<EventColumn>();
    // List<EventColumn> newColumns = new ArrayList<EventColumn>();
    // for (Column column : beforeColumns) {
    // oldColumns.add(copyEventColumn(column, true, tableHolder));
    // }
    //
    // for (Column column : afterColumns) {
    // newColumns.add(copyEventColumn(column, true, tableHolder));
    // }
    // Collections.sort(oldColumns, new EventColumnIndexComparable());
    // Collections.sort(newColumns, new EventColumnIndexComparable());
    // eventData.setOldKeys(oldColumns);// 做为老主键
    // eventData.setKeys(newColumns);// 做为新主键，需要保证新老主键字段数量一致
    // } else {
    // // 针对无主键，等同为所有都是主键进行处理
    // eventData.setKeys(columns);
    // }
    } else {
        throw new SelectException("this rowdata has no pks , entry: " + entry.toString() + " and rowData: " + rowData);
    }
    return eventData;
}

Example 5 with Column

use of com.alibaba.otter.canal.protocol.CanalEntry.Column in project otter by alibaba.

the class MessageParser method checkCompatibleLoopback.

/**
     * 检查otter3.0的兼容表处理，主要部分业务使用了3.0的功能，需要考虑兼容支持，后续可删除
     */
private int checkCompatibleLoopback(Pipeline pipeline, RowData rowData) {
    // 检查_info字段
    // 首先检查下after记录，从无变有的过程，一般出现在事务头
    Column infokColumn = getColumnIgnoreCase(rowData.getAfterColumnsList(), compatibleMarkInfoColumn);
    // 匹配对应的channel id
    if (infokColumn != null && infokColumn.getValue().toUpperCase().endsWith(RETL_CLIENT_FLAG)) {
        return 1;
    }
    infokColumn = getColumnIgnoreCase(rowData.getBeforeColumnsList(), compatibleMarkInfoColumn);
    if (infokColumn != null && infokColumn.getValue().toUpperCase().endsWith(RETL_CLIENT_FLAG)) {
        return 1;
    }
    // 检查_id字段
    Column markColumn = getColumnIgnoreCase(rowData.getAfterColumnsList(), compatibleMarkIdentifierColumn);
    // 匹配对应的channel id
    if (markColumn != null && pipeline.getChannelId().equals(Long.parseLong(markColumn.getValue()))) {
        return 2;
    }
    markColumn = getColumnIgnoreCase(rowData.getBeforeColumnsList(), compatibleMarkIdentifierColumn);
    if (markColumn != null && pipeline.getChannelId().equals(Long.parseLong(markColumn.getValue()))) {
        return 2;
    }
    return 0;
}