Examples with TableIdNotFoundException com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException used on opensource projects

Example 1 with TableIdNotFoundException

use of com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException in project canal by alibaba.

the class AbstractEventParser method start.

public void start() {
    super.start();
    MDC.put("destination", destination);
    // 配置transaction buffer
    // 初始化缓冲队列
    // 设置buffer大小
    transactionBuffer.setBufferSize(transactionSize);
    transactionBuffer.start();
    // 构造bin log parser
    // 初始化一下BinLogParser
    binlogParser = buildParser();
    binlogParser.start();
    // 启动工作线程
    parseThread = new Thread(new Runnable() {

        public void run() {
            MDC.put("destination", String.valueOf(destination));
            ErosaConnection erosaConnection = null;
            boolean isMariaDB = false;
            while (running) {
                try {
                    // 开始执行replication
                    // 1. 构造Erosa连接
                    erosaConnection = buildErosaConnection();
                    // 2. 启动一个心跳线程
                    startHeartBeat(erosaConnection);
                    // 3. 执行dump前的准备工作
                    preDump(erosaConnection);
                    // 链接
                    erosaConnection.connect();
                    long queryServerId = erosaConnection.queryServerId();
                    if (queryServerId != 0) {
                        serverId = queryServerId;
                    }
                    if (erosaConnection instanceof MysqlConnection) {
                        isMariaDB = ((MysqlConnection) erosaConnection).isMariaDB();
                    }
                    // 4. 获取最后的位置信息
                    long start = System.currentTimeMillis();
                    logger.warn("---> begin to find start position, it will be long time for reset or first position");
                    EntryPosition position = findStartPosition(erosaConnection);
                    final EntryPosition startPosition = position;
                    if (startPosition == null) {
                        throw new PositionNotFoundException("can't find start position for " + destination);
                    }
                    if (!processTableMeta(startPosition)) {
                        throw new CanalParseException("can't find init table meta for " + destination + " with position : " + startPosition);
                    }
                    long end = System.currentTimeMillis();
                    logger.warn("---> find start position successfully, {}", startPosition.toString() + " cost : " + (end - start) + "ms , the next step is binlog dump");
                    // 重新链接，因为在找position过程中可能有状态，需要断开后重建
                    erosaConnection.reconnect();
                    final SinkFunction sinkHandler = new SinkFunction<EVENT>() {

                        private LogPosition lastPosition;

                        public boolean sink(EVENT event) {
                            try {
                                CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, false);
                                if (!running) {
                                    return false;
                                }
                                if (entry != null) {
                                    // 有正常数据流过，清空exception
                                    exception = null;
                                    transactionBuffer.add(entry);
                                    // 记录一下对应的positions
                                    this.lastPosition = buildLastPosition(entry);
                                    // 记录一下最后一次有数据的时间
                                    lastEntryTime = System.currentTimeMillis();
                                }
                                return running;
                            } catch (TableIdNotFoundException e) {
                                throw e;
                            } catch (Throwable e) {
                                if (e.getCause() instanceof TableIdNotFoundException) {
                                    throw (TableIdNotFoundException) e.getCause();
                                }
                                // 记录一下，出错的位点信息
                                processSinkError(e, this.lastPosition, startPosition.getJournalName(), startPosition.getPosition());
                                // 继续抛出异常，让上层统一感知
                                throw new CanalParseException(e);
                            }
                        }
                    };
                    // 4. 开始dump数据
                    if (parallel) {
                        // build stage processor
                        multiStageCoprocessor = buildMultiStageCoprocessor();
                        if (isGTIDMode() && StringUtils.isNotEmpty(startPosition.getGtid())) {
                            // 判断所属instance是否启用GTID模式，是的话调用ErosaConnection中GTID对应方法dump数据
                            GTIDSet gtidSet = parseGtidSet(startPosition.getGtid(), isMariaDB);
                            ((MysqlMultiStageCoprocessor) multiStageCoprocessor).setGtidSet(gtidSet);
                            multiStageCoprocessor.start();
                            erosaConnection.dump(gtidSet, multiStageCoprocessor);
                        } else {
                            multiStageCoprocessor.start();
                            if (StringUtils.isEmpty(startPosition.getJournalName()) && startPosition.getTimestamp() != null) {
                                erosaConnection.dump(startPosition.getTimestamp(), multiStageCoprocessor);
                            } else {
                                erosaConnection.dump(startPosition.getJournalName(), startPosition.getPosition(), multiStageCoprocessor);
                            }
                        }
                    } else {
                        if (isGTIDMode() && StringUtils.isNotEmpty(startPosition.getGtid())) {
                            // 判断所属instance是否启用GTID模式，是的话调用ErosaConnection中GTID对应方法dump数据
                            erosaConnection.dump(parseGtidSet(startPosition.getGtid(), isMariaDB), sinkHandler);
                        } else {
                            if (StringUtils.isEmpty(startPosition.getJournalName()) && startPosition.getTimestamp() != null) {
                                erosaConnection.dump(startPosition.getTimestamp(), sinkHandler);
                            } else {
                                erosaConnection.dump(startPosition.getJournalName(), startPosition.getPosition(), sinkHandler);
                            }
                        }
                    }
                } catch (TableIdNotFoundException e) {
                    exception = e;
                    // 特殊处理TableIdNotFound异常,出现这样的异常，一种可能就是起始的position是一个事务当中，导致tablemap
                    // Event时间没解析过
                    needTransactionPosition.compareAndSet(false, true);
                    logger.error(String.format("dump address %s has an error, retrying. caused by ", runningInfo.getAddress().toString()), e);
                } catch (Throwable e) {
                    processDumpError(e);
                    exception = e;
                    if (!running) {
                        if (!(e instanceof java.nio.channels.ClosedByInterruptException || e.getCause() instanceof java.nio.channels.ClosedByInterruptException)) {
                            throw new CanalParseException(String.format("dump address %s has an error, retrying. ", runningInfo.getAddress().toString()), e);
                        }
                    } else {
                        logger.error(String.format("dump address %s has an error, retrying. caused by ", runningInfo.getAddress().toString()), e);
                        sendAlarm(destination, ExceptionUtils.getFullStackTrace(e));
                    }
                    if (parserExceptionHandler != null) {
                        parserExceptionHandler.handle(e);
                    }
                } finally {
                    // 重新置为中断状态
                    Thread.interrupted();
                    // 关闭一下链接
                    afterDump(erosaConnection);
                    try {
                        if (erosaConnection != null) {
                            erosaConnection.disconnect();
                        }
                    } catch (IOException e1) {
                        if (!running) {
                            throw new CanalParseException(String.format("disconnect address %s has an error, retrying. ", runningInfo.getAddress().toString()), e1);
                        } else {
                            logger.error("disconnect address {} has an error, retrying., caused by ", runningInfo.getAddress().toString(), e1);
                        }
                    }
                }
                // 出异常了，退出sink消费，释放一下状态
                eventSink.interrupt();
                // 重置一下缓冲队列，重新记录数据
                transactionBuffer.reset();
                // 重新置位
                binlogParser.reset();
                if (multiStageCoprocessor != null && multiStageCoprocessor.isStart()) {
                    // 处理 RejectedExecutionException
                    try {
                        multiStageCoprocessor.stop();
                    } catch (Throwable t) {
                        logger.debug("multi processor rejected:", t);
                    }
                }
                if (running) {
                    // sleep一段时间再进行重试
                    try {
                        Thread.sleep(10000 + RandomUtils.nextInt(10000));
                    } catch (InterruptedException e) {
                    }
                }
            }
            MDC.remove("destination");
        }
    });
    parseThread.setUncaughtExceptionHandler(handler);
    parseThread.setName(String.format("destination = %s , address = %s , EventParser", destination, runningInfo == null ? null : runningInfo.getAddress()));
    parseThread.start();
}

Example 2 with TableIdNotFoundException

use of com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException in project canal by alibaba.

the class LogEventConvert method parseRowsEventForTableMeta.

public TableMeta parseRowsEventForTableMeta(RowsLogEvent event) {
    TableMapLogEvent table = event.getTable();
    if (table == null) {
        // tableId对应的记录不存在
        throw new TableIdNotFoundException("not found tableId:" + event.getTableId());
    }
    boolean isHeartBeat = isAliSQLHeartBeat(table.getDbName(), table.getTableName());
    boolean isRDSHeartBeat = tableMetaCache.isOnRDS() && isRDSHeartBeat(table.getDbName(), table.getTableName());
    String fullname = table.getDbName() + "." + table.getTableName();
    // check name filter
    if (nameFilter != null && !nameFilter.filter(fullname)) {
        return null;
    }
    if (nameBlackFilter != null && nameBlackFilter.filter(fullname)) {
        return null;
    }
    // if (isHeartBeat || isRDSHeartBeat) {
    // // 忽略rds模式的mysql.ha_health_check心跳数据
    // return null;
    // }
    TableMeta tableMeta = null;
    if (isRDSHeartBeat) {
        // 处理rds模式的mysql.ha_health_check心跳数据
        // 主要RDS的心跳表基本无权限,需要mock一个tableMeta
        FieldMeta idMeta = new FieldMeta("id", "bigint(20)", true, false, "0");
        FieldMeta typeMeta = new FieldMeta("type", "char(1)", false, true, "0");
        tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
    } else if (isHeartBeat) {
        // 处理alisql模式的test.heartbeat心跳数据
        // 心跳表基本无权限,需要mock一个tableMeta
        FieldMeta idMeta = new FieldMeta("id", "smallint(6)", false, true, null);
        FieldMeta typeMeta = new FieldMeta("ts", "int(11)", true, false, null);
        tableMeta = new TableMeta(table.getDbName(), table.getTableName(), Arrays.asList(idMeta, typeMeta));
    }
    EntryPosition position = createPosition(event.getHeader());
    if (tableMetaCache != null && tableMeta == null) {
        // 入错存在table meta
        tableMeta = getTableMeta(table.getDbName(), table.getTableName(), true, position);
        if (tableMeta == null) {
            if (!filterTableError) {
                throw new CanalParseException("not found [" + fullname + "] in db , pls check!");
            }
        }
    }
    return tableMeta;
}

Example 3 with TableIdNotFoundException

use of com.taobao.tddl.dbsync.binlog.exception.TableIdNotFoundException in project canal by alibaba.

the class RowsLogEvent method fillTable.

public final void fillTable(LogContext context) {
    table = context.getTable(tableId);
    if (table == null) {
        throw new TableIdNotFoundException("not found tableId:" + tableId);
    }
    // end of statement check:
    if ((flags & RowsLogEvent.STMT_END_F) != 0) {
        // Now is safe to clear ignored map (clear_tables will also
        // delete original table map events stored in the map).
        context.clearAllTables();
    }
    int jsonColumnCount = 0;
    int columnCnt = table.getColumnCnt();
    ColumnInfo[] columnInfo = table.getColumnInfo();
    for (int i = 0; i < columnCnt; i++) {
        ColumnInfo info = columnInfo[i];
        if (info.type == LogEvent.MYSQL_TYPE_JSON) {
            jsonColumnCount++;
        }
    }
    this.jsonColumnCount = jsonColumnCount;
}