Examples with CanalParseException com.alibaba.otter.canal.parse.exception.CanalParseException used on opensource projects

Example 11 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class LogEventConvert method parseOneRow.

private boolean parseOneRow(RowData.Builder rowDataBuilder, RowsLogEvent event, RowsLogBuffer buffer, BitSet cols, boolean isAfter, TableMeta tableMeta) throws UnsupportedEncodingException {
    int columnCnt = event.getTable().getColumnCnt();
    ColumnInfo[] columnInfo = event.getTable().getColumnInfo();
    boolean tableError = false;
    // check table fileds count，只能处理加字段
    boolean existRDSNoPrimaryKey = false;
    if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
        if (tableMetaCache.isOnRDS()) {
            // 特殊处理下RDS的场景
            List<FieldMeta> primaryKeys = tableMeta.getPrimaryFields();
            if (primaryKeys == null || primaryKeys.isEmpty()) {
                if (columnInfo.length == tableMeta.getFileds().size() + 1 && columnInfo[columnInfo.length - 1].type == LogEvent.MYSQL_TYPE_LONGLONG) {
                    existRDSNoPrimaryKey = true;
                }
            }
        }
        if (!existRDSNoPrimaryKey) {
            // online ddl增加字段操作步骤：
            // 1. 新增一张临时表，将需要做ddl表的数据全量导入
            // 2. 在老表上建立I/U/D的trigger，增量的将数据插入到临时表
            // 3. 锁住应用请求，将临时表rename为老表的名字，完成增加字段的操作
            // 尝试做一次reload，可能因为ddl没有正确解析，或者使用了类似online ddl的操作
            // 因为online ddl没有对应表名的alter语法，所以不会有clear cache的操作
            // 强制重新获取一次
            tableMeta = getTableMeta(event.getTable().getDbName(), event.getTable().getTableName(), false);
            if (tableMeta == null) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("not found [" + event.getTable().getDbName() + "." + event.getTable().getTableName() + "] in db , pls check!");
                }
            }
            // 在做一次判断
            if (tableMeta != null && columnInfo.length > tableMeta.getFileds().size()) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("column size is not match for table:" + tableMeta.getFullName() + "," + columnInfo.length + " vs " + tableMeta.getFileds().size());
                }
            }
        } else {
            logger.warn("[" + event.getTable().getDbName() + "." + event.getTable().getTableName() + "] is no primary key , skip alibaba_rds_row_id column");
        }
    }
    for (int i = 0; i < columnCnt; i++) {
        ColumnInfo info = columnInfo[i];
        // mysql 5.6开始支持nolob/mininal类型,并不一定记录所有的列,需要进行判断
        if (!cols.get(i)) {
            continue;
        }
        if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
            // 不解析最后一列
            buffer.nextValue(info.type, info.meta, false);
            continue;
        }
        Column.Builder columnBuilder = Column.newBuilder();
        FieldMeta fieldMeta = null;
        if (tableMeta != null && !tableError) {
            // 处理file meta
            fieldMeta = tableMeta.getFileds().get(i);
            columnBuilder.setName(fieldMeta.getColumnName());
            columnBuilder.setIsKey(fieldMeta.isKey());
            // 增加mysql type类型,issue 73
            columnBuilder.setMysqlType(fieldMeta.getColumnType());
        }
        columnBuilder.setIndex(i);
        columnBuilder.setIsNull(false);
        // fixed issue
        // https://github.com/alibaba/canal/issues/66，特殊处理binary/varbinary，不能做编码处理
        boolean isBinary = false;
        if (fieldMeta != null) {
            if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "VARBINARY")) {
                isBinary = true;
            } else if (StringUtils.containsIgnoreCase(fieldMeta.getColumnType(), "BINARY")) {
                isBinary = true;
            }
        }
        buffer.nextValue(info.type, info.meta, isBinary);
        if (existRDSNoPrimaryKey && i == columnCnt - 1 && info.type == LogEvent.MYSQL_TYPE_LONGLONG) {
            // 不解析最后一列
            continue;
        }
        int javaType = buffer.getJavaType();
        if (buffer.isNull()) {
            columnBuilder.setIsNull(true);
        } else {
            final Serializable value = buffer.getValue();
            // 处理各种类型
            switch(javaType) {
                case Types.INTEGER:
                case Types.TINYINT:
                case Types.SMALLINT:
                case Types.BIGINT:
                    // 处理unsigned类型
                    Number number = (Number) value;
                    if (fieldMeta != null && fieldMeta.isUnsigned() && number.longValue() < 0) {
                        switch(buffer.getLength()) {
                            case 1:
                                /* MYSQL_TYPE_TINY */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(TINYINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.SMALLINT;
                                break;
                            case 2:
                                /* MYSQL_TYPE_SHORT */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(SMALLINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.INTEGER;
                                break;
                            case 3:
                                /* MYSQL_TYPE_INT24 */
                                columnBuilder.setValue(String.valueOf(Integer.valueOf(MEDIUMINT_MAX_VALUE + number.intValue())));
                                // 往上加一个量级
                                javaType = Types.INTEGER;
                                break;
                            case 4:
                                /* MYSQL_TYPE_LONG */
                                columnBuilder.setValue(String.valueOf(Long.valueOf(INTEGER_MAX_VALUE + number.longValue())));
                                // 往上加一个量级
                                javaType = Types.BIGINT;
                                break;
                            case 8:
                                /* MYSQL_TYPE_LONGLONG */
                                columnBuilder.setValue(BIGINT_MAX_VALUE.add(BigInteger.valueOf(number.longValue())).toString());
                                // 往上加一个量级，避免执行出错
                                javaType = Types.DECIMAL;
                                break;
                        }
                    } else {
                        // 对象为number类型，直接valueof即可
                        columnBuilder.setValue(String.valueOf(value));
                    }
                    break;
                // float
                case Types.REAL:
                case // double
                Types.DOUBLE:
                    // 对象为number类型，直接valueof即可
                    columnBuilder.setValue(String.valueOf(value));
                    break;
                case // bit
                Types.BIT:
                    // 对象为number类型
                    columnBuilder.setValue(String.valueOf(value));
                    break;
                case Types.DECIMAL:
                    columnBuilder.setValue(((BigDecimal) value).toPlainString());
                    break;
                case Types.TIMESTAMP:
                // break;
                case Types.TIME:
                case Types.DATE:
                    // 需要处理year
                    columnBuilder.setValue(value.toString());
                    break;
                case Types.BINARY:
                case Types.VARBINARY:
                case Types.LONGVARBINARY:
                    // meta，按编码解析text
                    if (fieldMeta != null && isText(fieldMeta.getColumnType())) {
                        columnBuilder.setValue(new String((byte[]) value, charset));
                        javaType = Types.CLOB;
                    } else {
                        // byte数组，直接使用iso-8859-1保留对应编码，浪费内存
                        columnBuilder.setValue(new String((byte[]) value, ISO_8859_1));
                        javaType = Types.BLOB;
                    }
                    break;
                case Types.CHAR:
                case Types.VARCHAR:
                    columnBuilder.setValue(value.toString());
                    break;
                default:
                    columnBuilder.setValue(value.toString());
            }
        }
        columnBuilder.setSqlType(javaType);
        // 设置是否update的标记位
        columnBuilder.setUpdated(isAfter && isUpdate(rowDataBuilder.getBeforeColumnsList(), columnBuilder.getIsNull() ? null : columnBuilder.getValue(), i));
        if (isAfter) {
            rowDataBuilder.addAfterColumns(columnBuilder.build());
        } else {
            rowDataBuilder.addBeforeColumns(columnBuilder.build());
        }
    }
    return tableError;
}

Example 12 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlConnection method loadBinlogImage.

/**
     * 获取一下binlog image格式
     */
private void loadBinlogImage() {
    ResultSetPacket rs = null;
    try {
        rs = query("show variables like 'binlog_row_image'");
    } catch (IOException e) {
        throw new CanalParseException(e);
    }
    List<String> columnValues = rs.getFieldValues();
    if (columnValues == null || columnValues.size() != 2) {
        // 可能历时版本没有image特性
        binlogImage = BinlogImage.FULL;
    } else {
        binlogImage = BinlogImage.valuesOf(columnValues.get(1));
    }
    if (binlogFormat == null) {
        throw new IllegalStateException("unexpected binlog image query result:" + rs.getFieldValues());
    }
}

Example 13 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlConnection method loadBinlogChecksum.

/**
     * 获取主库checksum信息
     * https://dev.mysql.com/doc/refman/5.6/en/replication-options
     * -binary-log.html#option_mysqld_binlog-checksum
     */
private void loadBinlogChecksum() {
    ResultSetPacket rs = null;
    try {
        rs = query("select @master_binlog_checksum");
    } catch (IOException e) {
        throw new CanalParseException(e);
    }
    List<String> columnValues = rs.getFieldValues();
    if (columnValues != null && columnValues.size() >= 1 && columnValues.get(0) != null && columnValues.get(0).toUpperCase().equals("CRC32")) {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_CRC32;
    } else {
        binlogChecksum = LogEvent.BINLOG_CHECKSUM_ALG_OFF;
    }
}

Example 14 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlConnection method loadBinlogFormat.

/**
     * 获取一下binlog format格式
     */
private void loadBinlogFormat() {
    ResultSetPacket rs = null;
    try {
        rs = query("show variables like 'binlog_format'");
    } catch (IOException e) {
        throw new CanalParseException(e);
    }
    List<String> columnValues = rs.getFieldValues();
    if (columnValues == null || columnValues.size() != 2) {
        logger.warn("unexpected binlog format query result, this may cause unexpected result, so throw exception to request network to io shutdown.");
        throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
    }
    binlogFormat = BinlogFormat.valuesOf(columnValues.get(1));
    if (binlogFormat == null) {
        throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
    }
}

Example 15 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlEventParser method findStartPosition.

/**
     * 查询当前的binlog位置
     */
private EntryPosition findStartPosition(MysqlConnection mysqlConnection) {
    try {
        ResultSetPacket packet = mysqlConnection.query("show binlog events limit 1");
        List<String> fields = packet.getFieldValues();
        if (CollectionUtils.isEmpty(fields)) {
            throw new CanalParseException("command : 'show binlog events limit 1' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
        }
        EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
        return endPosition;
    } catch (IOException e) {
        throw new CanalParseException("command : 'show binlog events limit 1' has an error!", e);
    }
}