Examples with CanalParseException com.alibaba.otter.canal.parse.exception.CanalParseException used on opensource projects

Example 31 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlConnection method loadBinlogFormat.

/**
 * 获取一下binlog format格式
 */
private void loadBinlogFormat() {
    ResultSetPacket rs = null;
    try {
        rs = query("show variables like 'binlog_format'");
    } catch (IOException e) {
        throw new CanalParseException(e);
    }
    List<String> columnValues = rs.getFieldValues();
    if (columnValues == null || columnValues.size() != 2) {
        logger.warn("unexpected binlog format query result, this may cause unexpected result, so throw exception to request network to io shutdown.");
        throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
    }
    binlogFormat = BinlogFormat.valuesOf(columnValues.get(1));
    if (binlogFormat == null) {
        throw new IllegalStateException("unexpected binlog format query result:" + rs.getFieldValues());
    }
}

Example 32 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class BinlogDownloadQueue method tryOne.

public BinlogFile tryOne() throws Throwable {
    BinlogFile binlogFile = binlogList.poll();
    if (binlogFile == null) {
        throw new CanalParseException("download binlog is null");
    }
    download(binlogFile);
    hostId = binlogFile.getHostInstanceID();
    this.currentSize++;
    return binlogFile;
}

Example 33 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class DatabaseTableMeta method buildMemFromSnapshot.

private EntryPosition buildMemFromSnapshot(EntryPosition position) {
    try {
        MetaSnapshotDO snapshotDO = metaSnapshotDAO.findByTimestamp(destination, position.getTimestamp());
        if (snapshotDO == null) {
            return null;
        }
        String binlogFile = snapshotDO.getBinlogFile();
        Long binlogOffest = snapshotDO.getBinlogOffest();
        String binlogMasterId = snapshotDO.getBinlogMasterId();
        Long binlogTimestamp = snapshotDO.getBinlogTimestamp();
        EntryPosition snapshotPosition = new EntryPosition(binlogFile, binlogOffest == null ? 0l : binlogOffest, binlogTimestamp == null ? 0l : binlogTimestamp, Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId));
        // data存储为Map<String,String>，每个分库一套建表
        String sqlData = snapshotDO.getData();
        JSONObject jsonObj = JSON.parseObject(sqlData);
        for (Map.Entry entry : jsonObj.entrySet()) {
            // 记录到内存
            if (!memoryTableMeta.apply(snapshotPosition, ObjectUtils.toString(entry.getKey()), ObjectUtils.toString(entry.getValue()), null)) {
                return null;
            }
        }
        return snapshotPosition;
    } catch (Throwable e) {
        throw new CanalParseException("apply failed caused by : " + e.getMessage(), e);
    }
}

Example 34 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class DatabaseTableMeta method applyHistoryToDB.

private boolean applyHistoryToDB(EntryPosition position, String schema, String ddl, String extra) {
    Map<String, String> content = new HashMap<>();
    content.put("destination", destination);
    content.put("binlogFile", position.getJournalName());
    content.put("binlogOffest", String.valueOf(position.getPosition()));
    content.put("binlogMasterId", String.valueOf(position.getServerId()));
    content.put("binlogTimestamp", String.valueOf(position.getTimestamp()));
    content.put("useSchema", schema);
    if (content.isEmpty()) {
        throw new RuntimeException("apply failed caused by content is empty in applyHistoryToDB");
    }
    // 待补充
    List<DdlResult> ddlResults = DruidDdlParser.parse(ddl, schema);
    if (ddlResults.size() > 0) {
        DdlResult ddlResult = ddlResults.get(0);
        content.put("sqlSchema", ddlResult.getSchemaName());
        content.put("sqlTable", ddlResult.getTableName());
        content.put("sqlType", ddlResult.getType().name());
        content.put("sqlText", ddl);
        content.put("extra", extra);
    }
    MetaHistoryDO metaDO = new MetaHistoryDO();
    try {
        BeanUtils.populate(metaDO, content);
        // 会建立唯一约束,解决:
        // 1. 重复的binlog file+offest
        // 2. 重复的masterId+timestamp
        metaHistoryDAO.insert(metaDO);
    } catch (Throwable e) {
        if (isUkDuplicateException(e)) {
            // 忽略掉重复的位点
            logger.warn("dup apply for sql : " + ddl);
        } else {
            throw new CanalParseException("apply history to db failed caused by : " + e.getMessage(), e);
        }
    }
    return true;
}

Example 35 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class DatabaseTableMeta method applyHistoryOnMemory.

private boolean applyHistoryOnMemory(EntryPosition position, EntryPosition rollbackPosition) {
    try {
        List<MetaHistoryDO> metaHistoryDOList = metaHistoryDAO.findByTimestamp(destination, position.getTimestamp(), rollbackPosition.getTimestamp());
        if (metaHistoryDOList == null) {
            return true;
        }
        for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) {
            String binlogFile = metaHistoryDO.getBinlogFile();
            Long binlogOffest = metaHistoryDO.getBinlogOffest();
            String binlogMasterId = metaHistoryDO.getBinlogMasterId();
            Long binlogTimestamp = metaHistoryDO.getBinlogTimestamp();
            String useSchema = metaHistoryDO.getUseSchema();
            String sqlData = metaHistoryDO.getSqlText();
            EntryPosition snapshotPosition = new EntryPosition(binlogFile, binlogOffest == null ? 0L : binlogOffest, binlogTimestamp == null ? 0L : binlogTimestamp, Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId));
            // 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之
            if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) {
                continue;
            } else if (rollbackPosition.getServerId().equals(snapshotPosition.getServerId()) && snapshotPosition.compareTo(rollbackPosition) > 0) {
                continue;
            }
            // 记录到内存
            if (!memoryTableMeta.apply(snapshotPosition, useSchema, sqlData, null)) {
                return false;
            }
        }
        return metaHistoryDOList.size() > 0;
    } catch (Throwable e) {
        throw new CanalParseException("apply failed", e);
    }
}