Examples with FileLogFetcher com.taobao.tddl.dbsync.binlog.FileLogFetcher used on opensource projects

Example 1 with FileLogFetcher

use of com.taobao.tddl.dbsync.binlog.FileLogFetcher in project canal by alibaba.

the class LocalBinLogConnection method dump.

public void dump(long timestampMills, SinkFunction func) throws IOException {
    List<File> currentBinlogs = binlogs.currentBinlogs();
    File current = currentBinlogs.get(currentBinlogs.size() - 1);
    long timestampSeconds = timestampMills / 1000;
    String binlogFilename = null;
    long binlogFileOffset = 0;
    FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
    LogDecoder decoder = new LogDecoder();
    decoder.handle(LogEvent.FORMAT_DESCRIPTION_EVENT);
    decoder.handle(LogEvent.QUERY_EVENT);
    decoder.handle(LogEvent.XID_EVENT);
    LogContext context = new LogContext();
    try {
        fetcher.open(current);
        context.setLogPosition(new LogPosition(current.getName()));
        while (running) {
            boolean needContinue = true;
            String lastXidLogFilename = current.getName();
            long lastXidLogFileOffset = 0;
            binlogFilename = lastXidLogFilename;
            binlogFileOffset = lastXidLogFileOffset;
            while (fetcher.fetch()) {
                LogEvent event = decoder.decode(fetcher, context);
                if (event != null) {
                    checkServerId(event);
                    if (event.getWhen() > timestampSeconds) {
                        break;
                    }
                    needContinue = false;
                    if (LogEvent.QUERY_EVENT == event.getHeader().getType()) {
                        if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "BEGIN")) {
                            binlogFilename = lastXidLogFilename;
                            binlogFileOffset = lastXidLogFileOffset;
                        } else if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "COMMIT")) {
                            lastXidLogFilename = current.getName();
                            lastXidLogFileOffset = event.getLogPos();
                        }
                    } else if (LogEvent.XID_EVENT == event.getHeader().getType()) {
                        lastXidLogFilename = current.getName();
                        lastXidLogFileOffset = event.getLogPos();
                    } else if (LogEvent.FORMAT_DESCRIPTION_EVENT == event.getHeader().getType()) {
                        lastXidLogFilename = current.getName();
                        lastXidLogFileOffset = event.getLogPos();
                    }
                }
            }
            if (needContinue) {
                // 读取下一个
                // 关闭上一个文件
                fetcher.close();
                File nextFile = binlogs.getBefore(current);
                if (nextFile == null) {
                    break;
                }
                current = nextFile;
                fetcher.open(current);
                context.setLogPosition(new LogPosition(current.getName()));
            } else {
                // 跳出
                break;
            }
        }
    } finally {
        if (fetcher != null) {
            fetcher.close();
        }
    }
    dump(binlogFilename, binlogFileOffset, func);
}

Example 2 with FileLogFetcher

use of com.taobao.tddl.dbsync.binlog.FileLogFetcher in project canal by alibaba.

the class LocalBinLogConnection method dump.

@Override
public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException {
    File current = new File(directory, binlogfilename);
    if (!current.exists()) {
        throw new CanalParseException("binlog:" + binlogfilename + " is not found");
    }
    try (FileLogFetcher fetcher = new FileLogFetcher(bufferSize)) {
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        fetcher.open(current, binlogPosition);
        context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
        while (running) {
            boolean needContinue = true;
            LogEvent event = null;
            while (fetcher.fetch()) {
                event = decoder.decode(fetcher, context);
                if (event == null) {
                    continue;
                }
                checkServerId(event);
                if (!coprocessor.publish(event)) {
                    needContinue = false;
                    break;
                }
            }
            // 关闭上一个文件
            fetcher.close();
            parserFinish(binlogfilename);
            if (needContinue) {
                // 读取下一个
                File nextFile;
                if (needWait) {
                    nextFile = binlogs.waitForNextFile(current);
                } else {
                    nextFile = binlogs.getNextFile(current);
                }
                if (nextFile == null) {
                    break;
                }
                current = nextFile;
                fetcher.open(current);
                binlogfilename = nextFile.getName();
            } else {
                // 跳出
                break;
            }
        }
    } catch (InterruptedException e) {
        logger.warn("LocalBinLogConnection dump interrupted");
    }
}

Example 3 with FileLogFetcher

use of com.taobao.tddl.dbsync.binlog.FileLogFetcher in project canal by alibaba.

the class LocalBinLogConnection method dump.

public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
    File current = new File(directory, binlogfilename);
    try (FileLogFetcher fetcher = new FileLogFetcher(bufferSize)) {
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        fetcher.open(current, binlogPosition);
        context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
        while (running) {
            boolean needContinue = true;
            LogEvent event = null;
            while (fetcher.fetch()) {
                event = decoder.decode(fetcher, context);
                if (event == null) {
                    continue;
                }
                checkServerId(event);
                if (!func.sink(event)) {
                    needContinue = false;
                    break;
                }
            }
            // 关闭上一个文件
            fetcher.close();
            parserFinish(current.getName());
            if (needContinue) {
                // 读取下一个
                File nextFile;
                if (needWait) {
                    nextFile = binlogs.waitForNextFile(current);
                } else {
                    nextFile = binlogs.getNextFile(current);
                }
                if (nextFile == null) {
                    break;
                }
                current = nextFile;
                fetcher.open(current);
                context.setLogPosition(new LogPosition(nextFile.getName()));
            } else {
                // 跳出
                break;
            }
        }
    } catch (InterruptedException e) {
        logger.warn("LocalBinLogConnection dump interrupted");
    }
}

Example 4 with FileLogFetcher

use of com.taobao.tddl.dbsync.binlog.FileLogFetcher in project canal by alibaba.

the class LocalBinLogConnection method dump.

@Override
public void dump(long timestampMills, MultiStageCoprocessor coprocessor) throws IOException {
    List<File> currentBinlogs = binlogs.currentBinlogs();
    File current = currentBinlogs.get(currentBinlogs.size() - 1);
    long timestampSeconds = timestampMills / 1000;
    String binlogFilename = null;
    long binlogFileOffset = 0;
    FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
    LogDecoder decoder = new LogDecoder();
    decoder.handle(LogEvent.FORMAT_DESCRIPTION_EVENT);
    decoder.handle(LogEvent.QUERY_EVENT);
    decoder.handle(LogEvent.XID_EVENT);
    LogContext context = new LogContext();
    try {
        fetcher.open(current);
        context.setLogPosition(new LogPosition(current.getName()));
        while (running) {
            boolean needContinue = true;
            String lastXidLogFilename = current.getName();
            long lastXidLogFileOffset = 0;
            binlogFilename = lastXidLogFilename;
            binlogFileOffset = lastXidLogFileOffset;
            while (fetcher.fetch()) {
                LogEvent event = decoder.decode(fetcher, context);
                if (event != null) {
                    checkServerId(event);
                    if (event.getWhen() > timestampSeconds) {
                        break;
                    }
                    needContinue = false;
                    if (LogEvent.QUERY_EVENT == event.getHeader().getType()) {
                        if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "BEGIN")) {
                            binlogFilename = lastXidLogFilename;
                            binlogFileOffset = lastXidLogFileOffset;
                        } else if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "COMMIT")) {
                            lastXidLogFilename = current.getName();
                            lastXidLogFileOffset = event.getLogPos();
                        }
                    } else if (LogEvent.XID_EVENT == event.getHeader().getType()) {
                        lastXidLogFilename = current.getName();
                        lastXidLogFileOffset = event.getLogPos();
                    } else if (LogEvent.FORMAT_DESCRIPTION_EVENT == event.getHeader().getType()) {
                        lastXidLogFilename = current.getName();
                        lastXidLogFileOffset = event.getLogPos();
                    }
                }
            }
            if (needContinue) {
                // 读取下一个
                // 关闭上一个文件
                fetcher.close();
                File nextFile = binlogs.getBefore(current);
                if (nextFile == null) {
                    break;
                }
                current = nextFile;
                fetcher.open(current);
                context.setLogPosition(new LogPosition(current.getName()));
            } else {
                // 跳出
                break;
            }
        }
    } finally {
        if (fetcher != null) {
            fetcher.close();
        }
    }
    dump(binlogFilename, binlogFileOffset, coprocessor);
}