Examples with CanalParseException com.alibaba.otter.canal.parse.exception.CanalParseException used on opensource projects

Example 21 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlEventParser method findStartPosition.

/**
 * 查询当前的binlog位置
 */
private EntryPosition findStartPosition(MysqlConnection mysqlConnection) {
    try {
        ResultSetPacket packet = mysqlConnection.query("show binlog events limit 1");
        List<String> fields = packet.getFieldValues();
        if (CollectionUtils.isEmpty(fields)) {
            throw new CanalParseException("command : 'show binlog events limit 1' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
        }
        EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
        return endPosition;
    } catch (IOException e) {
        throw new CanalParseException("command : 'show binlog events limit 1' has an error!", e);
    }
}

Example 22 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlEventParser method preDump.

protected void preDump(ErosaConnection connection) {
    if (!(connection instanceof MysqlConnection)) {
        throw new CanalParseException("Unsupported connection type : " + connection.getClass().getSimpleName());
    }
    if (binlogParser != null && binlogParser instanceof LogEventConvert) {
        metaConnection = (MysqlConnection) connection.fork();
        try {
            metaConnection.connect();
        } catch (IOException e) {
            throw new CanalParseException(e);
        }
        if (supportBinlogFormats != null && supportBinlogFormats.length > 0) {
            BinlogFormat format = ((MysqlConnection) metaConnection).getBinlogFormat();
            boolean found = false;
            for (BinlogFormat supportFormat : supportBinlogFormats) {
                if (supportFormat != null && format == supportFormat) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                throw new CanalParseException("Unsupported BinlogFormat " + format);
            }
        }
        if (supportBinlogImages != null && supportBinlogImages.length > 0) {
            BinlogImage image = ((MysqlConnection) metaConnection).getBinlogImage();
            boolean found = false;
            for (BinlogImage supportImage : supportBinlogImages) {
                if (supportImage != null && image == supportImage) {
                    found = true;
                    break;
                }
            }
            if (!found) {
                throw new CanalParseException("Unsupported BinlogImage " + image);
            }
        }
        if (tableMetaTSDB != null && tableMetaTSDB instanceof DatabaseTableMeta) {
            ((DatabaseTableMeta) tableMetaTSDB).setConnection(metaConnection);
            ((DatabaseTableMeta) tableMetaTSDB).setFilter(eventFilter);
            ((DatabaseTableMeta) tableMetaTSDB).setBlackFilter(eventBlackFilter);
            ((DatabaseTableMeta) tableMetaTSDB).setSnapshotInterval(tsdbSnapshotInterval);
            ((DatabaseTableMeta) tableMetaTSDB).setSnapshotExpire(tsdbSnapshotExpire);
            ((DatabaseTableMeta) tableMetaTSDB).init(destination);
        }
        tableMetaCache = new TableMetaCache(metaConnection, tableMetaTSDB);
        ((LogEventConvert) binlogParser).setTableMetaCache(tableMetaCache);
    }
}

Example 23 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlEventParser method findServerId.

/**
 * 查询当前db的serverId信息
 */
private Long findServerId(MysqlConnection mysqlConnection) {
    try {
        ResultSetPacket packet = mysqlConnection.query("show variables like 'server_id'");
        List<String> fields = packet.getFieldValues();
        if (CollectionUtils.isEmpty(fields)) {
            throw new CanalParseException("command : show variables like 'server_id' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
        }
        return Long.valueOf(fields.get(1));
    } catch (IOException e) {
        throw new CanalParseException("command : show variables like 'server_id' has an error!", e);
    }
}

Example 24 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlEventParser method generateUniqueServerId.

private final long generateUniqueServerId() {
    try {
        // a=`echo $masterip|cut -d\. -f1`
        // b=`echo $masterip|cut -d\. -f2`
        // c=`echo $masterip|cut -d\. -f3`
        // d=`echo $masterip|cut -d\. -f4`
        // #server_id=`expr $a \* 256 \* 256 \* 256 + $b \* 256 \* 256 + $c
        // \* 256 + $d `
        // #server_id=$b$c$d
        // server_id=`expr $b \* 256 \* 256 + $c \* 256 + $d `
        InetAddress localHost = InetAddress.getLocalHost();
        byte[] addr = localHost.getAddress();
        int salt = (destination != null) ? destination.hashCode() : 0;
        return // NL
        ((0x7f & salt) << 24) + ((0xff & (int) addr[1]) << 16) + // NL
        ((0xff & (int) addr[2]) << 8) + (0xff & (int) addr[3]);
    } catch (UnknownHostException e) {
        throw new CanalParseException("Unknown host", e);
    }
}

Example 25 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlEventParser method findTransactionBeginPosition.

// 根据想要的position，可能这个position对应的记录为rowdata，需要找到事务头，避免丢数据
// 主要考虑一个事务执行时间可能会几秒种，如果仅仅按照timestamp相同，则可能会丢失事务的前半部分数据
private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition) throws IOException {
    // 针对开始的第一条为非Begin记录，需要从该binlog扫描
    final java.util.concurrent.atomic.AtomicLong preTransactionStartPosition = new java.util.concurrent.atomic.AtomicLong(0L);
    mysqlConnection.reconnect();
    mysqlConnection.seek(entryPosition.getJournalName(), 4L, entryPosition.getGtid(), new SinkFunction<LogEvent>() {

        private LogPosition lastPosition;

        public boolean sink(LogEvent event) {
            try {
                CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true);
                if (entry == null) {
                    return true;
                }
                // 记录一下transaction begin position
                if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) {
                    preTransactionStartPosition.set(entry.getHeader().getLogfileOffset());
                }
                if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) {
                    // 退出
                    return false;
                }
                lastPosition = buildLastPosition(entry);
            } catch (Exception e) {
                processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
                return false;
            }
            return running;
        }
    });
    // 判断一下找到的最接近position的事务头的位置
    if (preTransactionStartPosition.get() > entryPosition.getPosition()) {
        logger.error("preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data");
        throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data");
    }
    return preTransactionStartPosition.get();
}