Examples with CanalParseException com.alibaba.otter.canal.parse.exception.CanalParseException used on opensource projects

Example 26 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class LogEventConvert method parseTableMapEvent.

public void parseTableMapEvent(TableMapLogEvent event) {
    try {
        String charsetDbName = new String(event.getDbName().getBytes(ISO_8859_1), charset.name());
        event.setDbname(charsetDbName);
        String charsetTbName = new String(event.getTableName().getBytes(ISO_8859_1), charset.name());
        event.setTblname(charsetTbName);
    } catch (UnsupportedEncodingException e) {
        throw new CanalParseException(e);
    }
}

Example 27 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class LogEventConvert method processFilter.

private boolean processFilter(String queryString, DdlResult result) {
    String schemaName = result.getSchemaName();
    String tableName = result.getTableName();
    // 更新下table meta cache
    if (tableMetaCache != null && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) {
        // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
        for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) {
            String schemaName0 = renameResult.getSchemaName();
            String tableName0 = renameResult.getTableName();
            if (StringUtils.isNotEmpty(tableName0)) {
                // 如果解析到了正确的表信息，则根据全名进行清除
                tableMetaCache.clearTableMeta(schemaName0, tableName0);
            } else {
                // 如果无法解析正确的表信息，则根据schema进行清除
                tableMetaCache.clearTableMetaWithSchemaName(schemaName0);
            }
        }
    }
    // fixed issue https://github.com/alibaba/canal/issues/58
    if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX || result.getType() == EventType.DINDEX) {
        if (filterQueryDdl) {
            return true;
        }
        if (StringUtils.isEmpty(tableName) || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) {
            // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析
            throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: " + queryString + " , and DdlResult: " + result.toString());
        // return null;
        } else {
            // check name filter
            String name = schemaName + "." + tableName;
            if (nameFilter != null && !nameFilter.filter(name)) {
                if (result.getType() == EventType.RENAME) {
                    // rename校验只要源和目标满足一个就进行操作
                    if (nameFilter != null && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                        return true;
                    }
                } else {
                    // 其他情况返回null
                    return true;
                }
            }
            if (nameBlackFilter != null && nameBlackFilter.filter(name)) {
                if (result.getType() == EventType.RENAME) {
                    // rename校验只要源和目标满足一个就进行操作
                    if (nameBlackFilter != null && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                        return true;
                    }
                } else {
                    // 其他情况返回null
                    return true;
                }
            }
        }
    } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE || result.getType() == EventType.DELETE) {
        // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
        if (filterQueryDml) {
            return true;
        }
    } else if (filterQueryDcl) {
        return true;
    }
    return false;
}

Example 28 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class LogEventConvert method parseOneRow.

private boolean parseOneRow(RowData.Builder rowDataBuilder, RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException {
    int columnCnt = event.getTable().getColumnCnt();
    ColumnInfo[] columnInfo = event.getTable().getColumnInfo();
    // mysql8.0针对set @@global.binlog_row_metadata='FULL' 可以记录部分的metadata信息
    boolean existOptionalMetaData = event.getTable().isExistOptionalMetaData();
    boolean tableError = false;
    // check table fileds count，只能处理加字段
    boolean existRDSNoPrimaryKey = false;
    // 获取字段过滤条件
    List<String> fieldList = null;
    List<String> blackFieldList = null;
    if (tableMeta != null) {
        fieldList = fieldFilterMap.get(tableMeta.getFullName().toUpperCase());
        blackFieldList = fieldBlackFilterMap.get(tableMeta.getFullName().toUpperCase());
    }
    if (tableMeta != null && columnInfo.length > tableMeta.getFields().size()) {
        if (tableMetaCache.isOnRDS() || tableMetaCache.isOnPolarX()) {
            // 特殊处理下RDS的场景
            List<FieldMeta> primaryKeys = tableMeta.getPrimaryFields();
            if (primaryKeys == null || primaryKeys.isEmpty()) {
                if (columnInfo.length == tableMeta.getFields().size() + 1 && columnInfo[columnInfo.length - 1].type == LogEvent.MYSQL_TYPE_LONGLONG) {
                    existRDSNoPrimaryKey = true;
                }
            }
        }
        EntryPosition position = createPosition(event.getHeader());
        if (!existRDSNoPrimaryKey) {
            // online ddl增加字段操作步骤：
            // 1. 新增一张临时表，将需要做ddl表的数据全量导入
            // 2. 在老表上建立I/U/D的trigger，增量的将数据插入到临时表
            // 3. 锁住应用请求，将临时表rename为老表的名字，完成增加字段的操作
            // 尝试做一次reload，可能因为ddl没有正确解析，或者使用了类似online ddl的操作
            // 因为online ddl没有对应表名的alter语法，所以不会有clear cache的操作
            // 强制重新获取一次
            tableMeta = getTableMeta(event.getTable().getDbName(), event.getTable().getTableName(), false, position);
            if (tableMeta == null) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("not found [" + event.getTable().getDbName() + "." + event.getTable().getTableName() + "] in db , pls check!");
                }
            }
            // 在做一次判断
            if (tableMeta != null && columnInfo.length > tableMeta.getFields().size()) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("column size is not match for table:" + tableMeta.getFullName() + "," + columnInfo.length + " vs " + tableMeta.getFields().size());
                }
            }
        // } else {
        // logger.warn("[" + event.getTable().getDbName() + "." +
        // event.getTable().getTableName()
        // + "] is no primary key , skip alibaba_rds_row_id column");
        }
    }
    for (int i = 0; i < columnCnt; i++) {
        ColumnInfo info = columnInfo[i];
        // mysql 5.6开始支持nolob/mininal类型,并不一定记录所有的列,需要进行判断
        if (!cols.get(i)) {
            continue;
        }
        if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
            // 不解析最后一列
            String rdsRowIdColumnName = "__#alibaba_rds_row_id#__";
            if (tableMetaCache.isOnPolarX()) {
                rdsRowIdColumnName = "_drds_implicit_id_";
            }
            buffer.nextValue(rdsRowIdColumnName, i, info.type, info.meta, false);
            Column.Builder columnBuilder = Column.newBuilder();
            columnBuilder.setName(rdsRowIdColumnName);
            columnBuilder.setIsKey(true);
            columnBuilder.setMysqlType("bigint");
            columnBuilder.setIndex(i);
            columnBuilder.setIsNull(false);
            Serializable value = buffer.getValue();
            columnBuilder.setValue(value.toString());
            columnBuilder.setSqlType(Types.BIGINT);
            columnBuilder.setUpdated(false);
            if (needField(fieldList, blackFieldList, columnBuilder.getName())) {
                if (isAfter) {
                    rowDataBuilder.addAfterColumns(columnBuilder.build());
                } else {
                    rowDataBuilder.addBeforeColumns(columnBuilder.build());
                }
            }
            continue;
        }
        FieldMeta fieldMeta = null;
        if (tableMeta != null && !tableError) {
            // 处理file meta
            fieldMeta = tableMeta.getFields().get(i);
        }
        if (fieldMeta != null && existOptionalMetaData && tableMetaCache.isOnTSDB()) {
            // check column info
            boolean check = StringUtils.equalsIgnoreCase(fieldMeta.getColumnName(), info.name);
            check &= (fieldMeta.isUnsigned() == info.unsigned);
            check &= (fieldMeta.isNullable() == info.nullable);
            if (!check) {
                throw new CanalParseException("MySQL8.0 unmatch column metadata & pls submit issue , table : " + tableMeta.getFullName() + ", db fieldMeta : " + fieldMeta.toString() + " , binlog fieldMeta : " + info.toString() + " , on : " + event.getHeader().getLogFileName() + ":" + (event.getHeader().getLogPos() - event.getHeader().getEventLen()));
            }
        }
        Column.Builder columnBuilder = Column.newBuilder();
        if (fieldMeta != null) {
            columnBuilder.setName(fieldMeta.getColumnName());
            columnBuilder.setIsKey(fieldMeta.isKey());
            // 增加mysql type类型,issue 73
            columnBuilder.setMysqlType(fieldMeta.getColumnType());
        } else if (existOptionalMetaData) {
            columnBuilder.setName(info.name);
            columnBuilder.setIsKey(info.pk);
        // mysql8.0里没有mysql type类型
        // columnBuilder.setMysqlType(fieldMeta.getColumnType());
        }
        columnBuilder.setIndex(i);
        columnBuilder.setIsNull(false);
        // fixed issue
        // https://github.com/alibaba/canal/issues/66，特殊处理binary/varbinary，不能做编码处理
        boolean isBinary = false;
        if (fieldMeta != null) {
            if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "VARBINARY")) {
                isBinary = true;
            } else if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "BINARY")) {
                isBinary = true;
            }
        }
        buffer.nextValue(columnBuilder.getName(), i, info.type, info.meta, isBinary);
        int javaType = buffer.getJavaType();
        if (buffer.isNull()) {
            columnBuilder.setIsNull(true);
        } else {
            final Serializable value = buffer.getValue();
            // 处理各种类型
            switch(javaType) {
                case Types.INTEGER:
                case Types.TINYINT:
                case Types.SMALLINT:
                case Types.BIGINT:
                    // 处理unsigned类型
                    Number number = (Number) value;
                    boolean isUnsigned = (fieldMeta != null ? fieldMeta.isUnsigned() : (existOptionalMetaData ? info.unsigned : false));
                    if (isUnsigned && number.longValue() < 0) {
                        switch(buffer.getLength()) {
                            case 1:
                                /* MYSQL_TYPE_TINY */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.SMALLINT;
                                break;
                            case 2:
                                /* MYSQL_TYPE_SHORT */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(SMALLINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.INTEGER;
                                break;
                            case 3:
                                /* MYSQL_TYPE_INT24 */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(MEDIUMINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.INTEGER;
                                break;
                            case 4:
                                /* MYSQL_TYPE_LONG */
                                columnBuilder.setValue(String.valueOf(Long.valueOf(INTEGER_MAX_VALUE + number.longValue())));
                                // 往上加一个量级
                                javaType = Types.BIGINT;
                                break;
                            case 8:
                                /* MYSQL_TYPE_LONGLONG */
                                columnBuilder.setValue(BIGINT_MAX_VALUE.add(BigInteger.valueOf(number.longValue())).toString());
                                // 往上加一个量级，避免执行出错
                                javaType = Types.DECIMAL;
                                break;
                        }
                    } else {
                        // 对象为number类型，直接valueof即可
                        columnBuilder.setValue(String.valueOf(value));
                    }
                    break;
                // float
                case Types.REAL:
                case // double
                Types.DOUBLE:
                    // 对象为number类型，直接valueof即可
                    columnBuilder.setValue(String.valueOf(value));
                    break;
                case // bit
                Types.BIT:
                    // 对象为number类型
                    columnBuilder.setValue(String.valueOf(value));
                    break;
                case Types.DECIMAL:
                    columnBuilder.setValue(((BigDecimal) value).toPlainString());
                    break;
                case Types.TIMESTAMP:
                // break;
                case Types.TIME:
                case Types.DATE:
                    // 需要处理year
                    columnBuilder.setValue(value.toString());
                    break;
                case Types.BINARY:
                case Types.VARBINARY:
                case Types.LONGVARBINARY:
                    // meta，按编码解析text
                    if (fieldMeta != null && isText(fieldMeta.getColumnType())) {
                        columnBuilder.setValue(new String((byte[]) value, charset));
                        javaType = Types.CLOB;
                    } else {
                        // byte数组，直接使用iso-8859-1保留对应编码，浪费内存
                        columnBuilder.setValue(new String((byte[]) value, ISO_8859_1));
                        // columnBuilder.setValueBytes(ByteString.copyFrom((byte[])
                        // value));
                        javaType = Types.BLOB;
                    }
                    break;
                case Types.CHAR:
                case Types.VARCHAR:
                    columnBuilder.setValue(value.toString());
                    break;
                default:
                    columnBuilder.setValue(value.toString());
            }
        }
        columnBuilder.setSqlType(javaType);
        // 设置是否update的标记位
        columnBuilder.setUpdated(isAfter && isUpdate(rowDataBuilder.getBeforeColumnsList(), columnBuilder.getIsNull() ? null : columnBuilder.getValue(), i));
        if (needField(fieldList, blackFieldList, columnBuilder.getName())) {
            if (isAfter) {
                rowDataBuilder.addAfterColumns(columnBuilder.build());
            } else {
                rowDataBuilder.addBeforeColumns(columnBuilder.build());
            }
        }
    }
    return tableError;
}

Example 29 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class LogEventConvert method parseRowsEvent.

public Entry parseRowsEvent(RowsLogEvent event, TableMeta tableMeta) {
    if (filterRows) {
        return null;
    }
    try {
        if (tableMeta == null) {
            // 如果没有外部指定
            tableMeta = parseRowsEventForTableMeta(event);
        }
        if (tableMeta == null) {
            // 拿不到表结构,执行忽略
            return null;
        }
        EventType eventType = null;
        int type = event.getHeader().getType();
        if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
            eventType = EventType.INSERT;
        } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type || LogEvent.PARTIAL_UPDATE_ROWS_EVENT == type) {
            eventType = EventType.UPDATE;
        } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
            eventType = EventType.DELETE;
        } else {
            throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
        }
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        rowChangeBuider.setTableId(event.getTableId());
        rowChangeBuider.setIsDdl(false);
        rowChangeBuider.setEventType(eventType);
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        boolean tableError = false;
        int rowsCount = 0;
        while (buffer.nextOneRow(columns, false)) {
            // 处理row记录
            RowData.Builder rowDataBuilder = RowData.newBuilder();
            if (EventType.INSERT == eventType) {
                // insert的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
            } else if (EventType.DELETE == eventType) {
                // delete的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
            } else {
                // update需要处理before/after
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
                if (!buffer.nextOneRow(changeColumns, true)) {
                    rowChangeBuider.addRowDatas(rowDataBuilder.build());
                    break;
                }
                tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
            }
            rowsCount++;
            rowChangeBuider.addRowDatas(rowDataBuilder.build());
        }
        TableMapLogEvent table = event.getTable();
        Header header = createHeader(event.getHeader(), table.getDbName(), table.getTableName(), eventType, rowsCount);
        RowChange rowChange = rowChangeBuider.build();
        if (tableError) {
            Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
            logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
            return null;
        } else {
            Entry entry = createEntry(header, EntryType.ROWDATA, rowChange.toByteString());
            return entry;
        }
    } catch (Exception e) {
        throw new CanalParseException("parse row data failed.", e);
    }
}

Example 30 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlConnection method loadBinlogImage.

/**
 * 获取一下binlog image格式
 */
private void loadBinlogImage() {
    ResultSetPacket rs = null;
    try {
        rs = query("show variables like 'binlog_row_image'");
    } catch (IOException e) {
        throw new CanalParseException(e);
    }
    List<String> columnValues = rs.getFieldValues();
    if (columnValues == null || columnValues.size() != 2) {
        // 可能历时版本没有image特性
        binlogImage = BinlogImage.FULL;
    } else {
        binlogImage = BinlogImage.valuesOf(columnValues.get(1));
    }
    if (binlogFormat == null) {
        throw new IllegalStateException("unexpected binlog image query result:" + rs.getFieldValues());
    }
}