Examples with LogEvent com.taobao.tddl.dbsync.binlog.LogEvent used on opensource projects

Example 1 with LogEvent

use of com.taobao.tddl.dbsync.binlog.LogEvent in project canal by alibaba.

the class MysqlEventParser method findAsPerTimestampInSpecificLogFile.

/**
     * 根据给定的时间戳，在指定的binlog中找到最接近于该时间戳(必须是小于时间戳)的一个事务起始位置。
     * 针对最后一个binlog会给定endPosition，避免无尽的查询
     */
private EntryPosition findAsPerTimestampInSpecificLogFile(MysqlConnection mysqlConnection, final Long startTimestamp, final EntryPosition endPosition, final String searchBinlogFile) {
    final LogPosition logPosition = new LogPosition();
    try {
        mysqlConnection.reconnect();
        // 开始遍历文件
        mysqlConnection.seek(searchBinlogFile, 4L, new SinkFunction<LogEvent>() {

            private LogPosition lastPosition;

            public boolean sink(LogEvent event) {
                EntryPosition entryPosition = null;
                try {
                    CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
                    if (entry == null) {
                        return true;
                    }
                    String logfilename = entry.getHeader().getLogfileName();
                    Long logfileoffset = entry.getHeader().getLogfileOffset();
                    Long logposTimestamp = entry.getHeader().getExecuteTime();
                    if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType()) || CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
                        logger.debug("compare exit condition:{},{},{}, startTimestamp={}...", new Object[] { logfilename, logfileoffset, logposTimestamp, startTimestamp });
                        // 事务头和尾寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出
                        if (logposTimestamp >= startTimestamp) {
                            return false;
                        }
                    }
                    if (StringUtils.equals(endPosition.getJournalName(), logfilename) && endPosition.getPosition() <= (logfileoffset + event.getEventLen())) {
                        return false;
                    }
                    // data.length，代表该事务的下一条offest，避免多余的事务重复
                    if (CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
                        entryPosition = new EntryPosition(logfilename, logfileoffset + event.getEventLen(), logposTimestamp);
                        logger.debug("set {} to be pending start position before finding another proper one...", entryPosition);
                        logPosition.setPostion(entryPosition);
                    } else if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) {
                        // 当前事务开始位点
                        entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp);
                        logger.debug("set {} to be pending start position before finding another proper one...", entryPosition);
                        logPosition.setPostion(entryPosition);
                    }
                    lastPosition = buildLastPosition(entry);
                } catch (Throwable e) {
                    processSinkError(e, lastPosition, searchBinlogFile, 4L);
                }
                return running;
            }
        });
    } catch (IOException e) {
        logger.error("ERROR ## findAsPerTimestampInSpecificLogFile has an error", e);
    }
    if (logPosition.getPostion() != null) {
        return logPosition.getPostion();
    } else {
        return null;
    }
}

Example 2 with LogEvent

use of com.taobao.tddl.dbsync.binlog.LogEvent in project canal by alibaba.

the class LocalBinLogConnection method dump.

public void dump(long timestampMills, SinkFunction func) throws IOException {
    List<File> currentBinlogs = binlogs.currentBinlogs();
    File current = currentBinlogs.get(currentBinlogs.size() - 1);
    long timestampSeconds = timestampMills / 1000;
    String binlogFilename = null;
    long binlogFileOffset = 0;
    FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
    LogDecoder decoder = new LogDecoder();
    decoder.handle(LogEvent.FORMAT_DESCRIPTION_EVENT);
    decoder.handle(LogEvent.QUERY_EVENT);
    decoder.handle(LogEvent.XID_EVENT);
    LogContext context = new LogContext();
    try {
        fetcher.open(current);
        context.setLogPosition(new LogPosition(current.getName()));
        while (running) {
            boolean needContinue = true;
            String lastXidLogFilename = current.getName();
            long lastXidLogFileOffset = 0;
            binlogFilename = lastXidLogFilename;
            binlogFileOffset = lastXidLogFileOffset;
            while (fetcher.fetch()) {
                LogEvent event = decoder.decode(fetcher, context);
                if (event != null) {
                    checkServerId(event);
                    if (event.getWhen() > timestampSeconds) {
                        break;
                    }
                    needContinue = false;
                    if (LogEvent.QUERY_EVENT == event.getHeader().getType()) {
                        if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "BEGIN")) {
                            binlogFilename = lastXidLogFilename;
                            binlogFileOffset = lastXidLogFileOffset;
                        } else if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "COMMIT")) {
                            lastXidLogFilename = current.getName();
                            lastXidLogFileOffset = event.getLogPos();
                        }
                    } else if (LogEvent.XID_EVENT == event.getHeader().getType()) {
                        lastXidLogFilename = current.getName();
                        lastXidLogFileOffset = event.getLogPos();
                    } else if (LogEvent.FORMAT_DESCRIPTION_EVENT == event.getHeader().getType()) {
                        lastXidLogFilename = current.getName();
                        lastXidLogFileOffset = event.getLogPos();
                    }
                }
            }
            if (needContinue) {
                // 读取下一个
                // 关闭上一个文件
                fetcher.close();
                File nextFile = binlogs.getBefore(current);
                if (nextFile == null) {
                    break;
                }
                current = nextFile;
                fetcher.open(current);
                context.setLogPosition(new LogPosition(current.getName()));
            } else {
                // 跳出
                break;
            }
        }
    } finally {
        if (fetcher != null) {
            fetcher.close();
        }
    }
    dump(binlogFilename, binlogFileOffset, func);
}

Example 3 with LogEvent

use of com.taobao.tddl.dbsync.binlog.LogEvent in project canal by alibaba.

the class LocalBinLogConnection method dump.

@Override
public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException {
    File current = new File(directory, binlogfilename);
    if (!current.exists()) {
        throw new CanalParseException("binlog:" + binlogfilename + " is not found");
    }
    try (FileLogFetcher fetcher = new FileLogFetcher(bufferSize)) {
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        fetcher.open(current, binlogPosition);
        context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
        while (running) {
            boolean needContinue = true;
            LogEvent event = null;
            while (fetcher.fetch()) {
                event = decoder.decode(fetcher, context);
                if (event == null) {
                    continue;
                }
                checkServerId(event);
                if (!coprocessor.publish(event)) {
                    needContinue = false;
                    break;
                }
            }
            // 关闭上一个文件
            fetcher.close();
            parserFinish(binlogfilename);
            if (needContinue) {
                // 读取下一个
                File nextFile;
                if (needWait) {
                    nextFile = binlogs.waitForNextFile(current);
                } else {
                    nextFile = binlogs.getNextFile(current);
                }
                if (nextFile == null) {
                    break;
                }
                current = nextFile;
                fetcher.open(current);
                binlogfilename = nextFile.getName();
            } else {
                // 跳出
                break;
            }
        }
    } catch (InterruptedException e) {
        logger.warn("LocalBinLogConnection dump interrupted");
    }
}

Example 4 with LogEvent

use of com.taobao.tddl.dbsync.binlog.LogEvent in project canal by alibaba.

the class MysqlConnection method dump.

@Override
public void dump(GTIDSet gtidSet, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendBinlogDumpGTID(gtidSet);
    try (DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize())) {
        fetcher.start(connector.getChannel());
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
        // fix bug: #890 将gtid传输至context中，供decode使用
        context.setGtidSet(gtidSet);
        while (fetcher.fetch()) {
            accumulateReceivedBytes(fetcher.limit());
            LogEvent event = null;
            event = decoder.decode(fetcher, context);
            if (event == null) {
                throw new CanalParseException("parse failed");
            }
            if (!func.sink(event)) {
                break;
            }
        }
    }
}

Example 5 with LogEvent

use of com.taobao.tddl.dbsync.binlog.LogEvent in project canal by alibaba.

the class MysqlConnection method seek.

/**
 * 加速主备切换时的查找速度，做一些特殊优化，比如只解析事务头或者尾
 */
public void seek(String binlogfilename, Long binlogPosition, String gtid, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendBinlogDump(binlogfilename, binlogPosition);
    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    fetcher.start(connector.getChannel());
    LogDecoder decoder = new LogDecoder();
    decoder.handle(LogEvent.ROTATE_EVENT);
    decoder.handle(LogEvent.FORMAT_DESCRIPTION_EVENT);
    decoder.handle(LogEvent.QUERY_EVENT);
    decoder.handle(LogEvent.XID_EVENT);
    LogContext context = new LogContext();
    // using CHANGE MASTER TO MASTER_AUTO_POSITION = 1 ...
    if (StringUtils.isNotEmpty(gtid)) {
        GTIDSet gtidSet = parseGtidSet(gtid, isMariaDB());
        if (isMariaDB()) {
            decoder.handle(LogEvent.GTID_EVENT);
            decoder.handle(LogEvent.GTID_LIST_EVENT);
        } else {
            decoder.handle(LogEvent.GTID_LOG_EVENT);
        }
        context.setGtidSet(gtidSet);
    }
    context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
    while (fetcher.fetch()) {
        accumulateReceivedBytes(fetcher.limit());
        LogEvent event = null;
        event = decoder.decode(fetcher, context);
        if (event == null) {
            throw new CanalParseException("parse failed");
        }
        if (!func.sink(event)) {
            break;
        }
    }
}