Examples with CanalParseException com.alibaba.otter.canal.parse.exception.CanalParseException used on opensource projects

Example 6 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlEventParser method preDump.

protected void preDump(ErosaConnection connection) {
    if (!(connection instanceof MysqlConnection)) {
        throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
    }
    if (binlogParser != null && binlogParser instanceof LogEventConvert) {
        metaConnection = (MysqlConnection) connection.fork();
        try {
            metaConnection.connect();
        } catch (IOException e) {
            throw new CanalParseException(e);
        }
        if (supportBinlogFormats != null && supportBinlogFormats.length > 0) {
            BinlogFormat format = ((MysqlConnection) metaConnection).getBinlogFormat();
            boolean found = false;
            for (BinlogFormat supportFormat : supportBinlogFormats) {
                if (supportFormat != null && format == supportFormat) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                throw new CanalParseException("Unsupported BinlogFormat " + format);
            }
        }
        if (supportBinlogImages != null && supportBinlogImages.length > 0) {
            BinlogImage image = ((MysqlConnection) metaConnection).getBinlogImage();
            boolean found = false;
            for (BinlogImage supportImage : supportBinlogImages) {
                if (supportImage != null && image == supportImage) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                throw new CanalParseException("Unsupported BinlogImage " + image);
            }
        }
        tableMetaCache = new TableMetaCache(metaConnection);
        ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
    }
}

Example 7 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlEventParser method findEndPosition.

/**
     * 查询当前的binlog位置
     */
private EntryPosition findEndPosition(MysqlConnection mysqlConnection) {
    try {
        ResultSetPacket packet = mysqlConnection.query("show master status");
        List<String> fields = packet.getFieldValues();
        if (CollectionUtils.isEmpty(fields)) {
            throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
        }
        EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
        return endPosition;
    } catch (IOException e) {
        throw new CanalParseException("command : 'show master status' has an error!", e);
    }
}

Example 8 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlEventParser method findTransactionBeginPosition.

// 根据想要的position，可能这个position对应的记录为rowdata，需要找到事务头，避免丢数据
// 主要考虑一个事务执行时间可能会几秒种，如果仅仅按照timestamp相同，则可能会丢失事务的前半部分数据
private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition) throws IOException {
    // 尝试找到一个合适的位置
    final AtomicBoolean reDump = new AtomicBoolean(false);
    mysqlConnection.reconnect();
    mysqlConnection.seek(entryPosition.getJournalName(), entryPosition.getPosition(), new SinkFunction<LogEvent>() {

        private LogPosition lastPosition;

        public boolean sink(LogEvent event) {
            try {
                CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
                if (entry == null) {
                    return true;
                }
                // 直接查询第一条业务数据，确认是否为事务Begin/End
                if (CanalEntry.EntryType.TRANSACTIONBEGIN == entry.getEntryType() || CanalEntry.EntryType.TRANSACTIONEND == entry.getEntryType()) {
                    lastPosition = buildLastPosition(entry);
                    return false;
                } else {
                    reDump.set(true);
                    lastPosition = buildLastPosition(entry);
                    return false;
                }
            } catch (Exception e) {
                // 上一次记录的poistion可能为一条update/insert/delete变更事件，直接进行dump的话，会缺少tableMap事件，导致tableId未进行解析
                processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
                reDump.set(true);
                return false;
            }
        }
    });
    // 针对开始的第一条为非Begin记录，需要从该binlog扫描
    if (reDump.get()) {
        final AtomicLong preTransactionStartPosition = new AtomicLong(0L);
        mysqlConnection.reconnect();
        mysqlConnection.seek(entryPosition.getJournalName(), 4L, new SinkFunction<LogEvent>() {

            private LogPosition lastPosition;

            public boolean sink(LogEvent event) {
                try {
                    CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
                    if (entry == null) {
                        return true;
                    }
                    // 记录一下transaction begin position
                    if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) {
                        preTransactionStartPosition.set(entry.getHeader().getLogfileOffset());
                    }
                    if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) {
                        // 退出
                        return false;
                    }
                    lastPosition = buildLastPosition(entry);
                } catch (Exception e) {
                    processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
                    return false;
                }
                return running;
            }
        });
        // 判断一下找到的最接近position的事务头的位置
        if (preTransactionStartPosition.get() > entryPosition.getPosition()) {
            logger.error("preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data");
            throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data");
        }
        return preTransactionStartPosition.get();
    } else {
        return entryPosition.getPosition();
    }
}

Example 9 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class LogEventConvert method parseRowsEvent.

private Entry parseRowsEvent(RowsLogEvent event) {
    if (filterRows) {
        return null;
    }
    try {
        TableMapLogEvent table = event.getTable();
        if (table == null) {
            // tableId对应的记录不存在
            throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
        }
        String fullname = table.getDbName() + "." + table.getTableName();
        // check name filter
        if (nameFilter != null && !nameFilter.filter(fullname)) {
            return null;
        }
        if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
            return null;
        }
        if (tableMetaCache.isOnRDS() && "mysql.ha_health_check".equals(fullname)) {
            // 忽略rds模式的mysql.ha_health_check心跳数据
            return null;
        }
        EventType eventType = null;
        int type = event.getHeader().getType();
        if (LogEvent.WRITE_ROWS_EVENT_V1 == type || LogEvent.WRITE_ROWS_EVENT == type) {
            eventType = EventType.INSERT;
        } else if (LogEvent.UPDATE_ROWS_EVENT_V1 == type || LogEvent.UPDATE_ROWS_EVENT == type) {
            eventType = EventType.UPDATE;
        } else if (LogEvent.DELETE_ROWS_EVENT_V1 == type || LogEvent.DELETE_ROWS_EVENT == type) {
            eventType = EventType.DELETE;
        } else {
            throw new CanalParseException("unsupport event type :" + event.getHeader().getType());
        }
        Header header = createHeader(binlogFileName, event.getHeader(), table.getDbName(), table.getTableName(), eventType);
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        rowChangeBuider.setTableId(event.getTableId());
        rowChangeBuider.setIsDdl(false);
        rowChangeBuider.setEventType(eventType);
        RowsLogBuffer buffer = event.getRowsBuf(charset.name());
        BitSet columns = event.getColumns();
        BitSet changeColumns = event.getChangeColumns();
        boolean tableError = false;
        TableMeta tableMeta = null;
        if (tableMetaCache != null) {
            // 入错存在table meta cache
            tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true);
            if (tableMeta == null) {
                tableError = true;
                if (!filterTableError) {
                    throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
                }
            }
        }
        while (buffer.nextOneRow(columns)) {
            // 处理row记录
            RowData.Builder rowDataBuilder = RowData.newBuilder();
            if (EventType.INSERT == eventType) {
                // insert的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, true, tableMeta);
            } else if (EventType.DELETE == eventType) {
                // delete的记录放在before字段中
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
            } else {
                // update需要处理before/after
                tableError |= parseOneRow(rowDataBuilder, event, buffer, columns, false, tableMeta);
                if (!buffer.nextOneRow(changeColumns)) {
                    rowChangeBuider.addRowDatas(rowDataBuilder.build());
                    break;
                }
                tableError |= parseOneRow(rowDataBuilder, event, buffer, changeColumns, true, tableMeta);
            }
            rowChangeBuider.addRowDatas(rowDataBuilder.build());
        }
        RowChange rowChange = rowChangeBuider.build();
        if (tableError) {
            Entry entry = createEntry(header, EntryType.ROWDATA, ByteString.EMPTY);
            logger.warn("table parser error : {}storeValue: {}", entry.toString(), rowChange.toString());
            return null;
        } else {
            Entry entry = createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
            return entry;
        }
    } catch (Exception e) {
        throw new CanalParseException("parse row data failed.", e);
    }
}

Example 10 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class LogEventConvert method parseQueryEvent.

private Entry parseQueryEvent(QueryLogEvent event) {
    String queryString = event.getQuery();
    if (StringUtils.endsWithIgnoreCase(queryString, BEGIN)) {
        TransactionBegin transactionBegin = createTransactionBegin(event.getSessionId());
        Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
        return createEntry(header, EntryType.TRANSACTIONBEGIN, transactionBegin.toByteString());
    } else if (StringUtils.endsWithIgnoreCase(queryString, COMMIT)) {
        // MyISAM可能不会有xid事件
        TransactionEnd transactionEnd = createTransactionEnd(0L);
        Header header = createHeader(binlogFileName, event.getHeader(), "", "", null);
        return createEntry(header, EntryType.TRANSACTIONEND, transactionEnd.toByteString());
    } else {
        // DDL语句处理
        DdlResult result = SimpleDdlParser.parse(queryString, event.getDbName());
        String schemaName = event.getDbName();
        if (StringUtils.isNotEmpty(result.getSchemaName())) {
            schemaName = result.getSchemaName();
        }
        String tableName = result.getTableName();
        EventType type = EventType.QUERY;
        // fixed issue https://github.com/alibaba/canal/issues/58
        if (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.CREATE || result.getType() == EventType.TRUNCATE || result.getType() == EventType.RENAME || result.getType() == EventType.CINDEX || result.getType() == EventType.DINDEX) {
            if (filterQueryDdl) {
                return null;
            }
            type = result.getType();
            if (StringUtils.isEmpty(tableName) || (result.getType() == EventType.RENAME && StringUtils.isEmpty(result.getOriTableName()))) {
                // 如果解析不出tableName,记录一下日志，方便bugfix，目前直接抛出异常，中断解析
                throw new CanalParseException("SimpleDdlParser process query failed. pls submit issue with this queryString: " + queryString + " , and DdlResult: " + result.toString());
            // return null;
            } else {
                // check name filter
                String name = schemaName + "." + tableName;
                if (nameFilter != null && !nameFilter.filter(name)) {
                    if (result.getType() == EventType.RENAME) {
                        // rename校验只要源和目标满足一个就进行操作
                        if (nameFilter != null && !nameFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                            return null;
                        }
                    } else {
                        // 其他情况返回null
                        return null;
                    }
                }
                if (nameBlackFilter != null && nameBlackFilter.filter(name)) {
                    if (result.getType() == EventType.RENAME) {
                        // rename校验只要源和目标满足一个就进行操作
                        if (nameBlackFilter != null && nameBlackFilter.filter(result.getOriSchemaName() + "." + result.getOriTableName())) {
                            return null;
                        }
                    } else {
                        // 其他情况返回null
                        return null;
                    }
                }
            }
        } else if (result.getType() == EventType.INSERT || result.getType() == EventType.UPDATE || result.getType() == EventType.DELETE) {
            // 对外返回，保证兼容，还是返回QUERY类型，这里暂不解析tableName，所以无法支持过滤
            if (filterQueryDml) {
                return null;
            }
        } else if (filterQueryDcl) {
            return null;
        }
        // 更新下table meta cache
        if (tableMetaCache != null && (result.getType() == EventType.ALTER || result.getType() == EventType.ERASE || result.getType() == EventType.RENAME)) {
            for (DdlResult renameResult = result; renameResult != null; renameResult = renameResult.getRenameTableResult()) {
                // 防止rename语句后产生schema变更带来影响
                String schemaName0 = event.getDbName();
                if (StringUtils.isNotEmpty(renameResult.getSchemaName())) {
                    schemaName0 = renameResult.getSchemaName();
                }
                tableName = renameResult.getTableName();
                if (StringUtils.isNotEmpty(tableName)) {
                    // 如果解析到了正确的表信息，则根据全名进行清除
                    tableMetaCache.clearTableMeta(schemaName0, tableName);
                } else {
                    // 如果无法解析正确的表信息，则根据schema进行清除
                    tableMetaCache.clearTableMetaWithSchemaName(schemaName0);
                }
            }
        }
        Header header = createHeader(binlogFileName, event.getHeader(), schemaName, tableName, type);
        RowChange.Builder rowChangeBuider = RowChange.newBuilder();
        if (result.getType() != EventType.QUERY) {
            rowChangeBuider.setIsDdl(true);
        }
        rowChangeBuider.setSql(queryString);
        if (StringUtils.isNotEmpty(event.getDbName())) {
            // 可能为空
            rowChangeBuider.setDdlSchemaName(event.getDbName());
        }
        rowChangeBuider.setEventType(result.getType());
        return createEntry(header, EntryType.ROWDATA, rowChangeBuider.build().toByteString());
    }
}