Examples with LogDecoder com.taobao.tddl.dbsync.binlog.LogDecoder used on opensource projects

Example 1 with LogDecoder

use of com.taobao.tddl.dbsync.binlog.LogDecoder in project canal by alibaba.

the class DirectLogFetcherTest method testSimple.

@Test
public void testSimple() {
    DirectLogFetcher fetcher = new DirectLogFetcher();
    try {
        MysqlConnector connector = new MysqlConnector(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx");
        connector.connect();
        sendBinlogDump(connector, "mysql-bin.001016", 4L, 3);
        fetcher.start(connector.getChannel());
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        while (fetcher.fetch()) {
            LogEvent event = null;
            event = decoder.decode(fetcher, context);
            if (event == null) {
                throw new RuntimeException("parse failed");
            }
            int eventType = event.getHeader().getType();
            switch(eventType) {
                case LogEvent.ROTATE_EVENT:
                    // event).getFilename();
                    break;
                case LogEvent.WRITE_ROWS_EVENT_V1:
                case LogEvent.WRITE_ROWS_EVENT:
                    // parseRowsEvent((WriteRowsLogEvent) event);
                    break;
                case LogEvent.UPDATE_ROWS_EVENT_V1:
                case LogEvent.UPDATE_ROWS_EVENT:
                    // parseRowsEvent((UpdateRowsLogEvent) event);
                    break;
                case LogEvent.DELETE_ROWS_EVENT_V1:
                case LogEvent.DELETE_ROWS_EVENT:
                    // parseRowsEvent((DeleteRowsLogEvent) event);
                    break;
                case LogEvent.QUERY_EVENT:
                    // parseQueryEvent((QueryLogEvent) event);
                    break;
                case LogEvent.ROWS_QUERY_LOG_EVENT:
                    // parseRowsQueryEvent((RowsQueryLogEvent) event);
                    break;
                case LogEvent.ANNOTATE_ROWS_EVENT:
                    break;
                case LogEvent.XID_EVENT:
                    break;
                default:
                    break;
            }
        }
    } catch (Exception e) {
        e.printStackTrace();
        Assert.fail(e.getMessage());
    } finally {
        try {
            fetcher.close();
        } catch (IOException e) {
            Assert.fail(e.getMessage());
        }
    }
}

Example 2 with LogDecoder

use of com.taobao.tddl.dbsync.binlog.LogDecoder in project canal by alibaba.

the class LocalBinLogConnection method dump.

public void dump(long timestampMills, SinkFunction func) throws IOException {
    List<File> currentBinlogs = binlogs.currentBinlogs();
    File current = currentBinlogs.get(currentBinlogs.size() - 1);
    long timestampSeconds = timestampMills / 1000;
    String binlogFilename = null;
    long binlogFileOffset = 0;
    FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
    LogDecoder decoder = new LogDecoder();
    decoder.handle(LogEvent.FORMAT_DESCRIPTION_EVENT);
    decoder.handle(LogEvent.QUERY_EVENT);
    decoder.handle(LogEvent.XID_EVENT);
    LogContext context = new LogContext();
    try {
        fetcher.open(current);
        context.setLogPosition(new LogPosition(current.getName()));
        while (running) {
            boolean needContinue = true;
            String lastXidLogFilename = current.getName();
            long lastXidLogFileOffset = 0;
            binlogFilename = lastXidLogFilename;
            binlogFileOffset = lastXidLogFileOffset;
            L: while (fetcher.fetch()) {
                LogEvent event;
                do {
                    event = decoder.decode(fetcher, context);
                    if (event != null) {
                        if (event.getWhen() > timestampSeconds) {
                            break L;
                        }
                        needContinue = false;
                        if (LogEvent.QUERY_EVENT == event.getHeader().getType()) {
                            if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "BEGIN")) {
                                binlogFilename = lastXidLogFilename;
                                binlogFileOffset = lastXidLogFileOffset;
                            } else if (StringUtils.endsWithIgnoreCase(((QueryLogEvent) event).getQuery(), "COMMIT")) {
                                lastXidLogFilename = current.getName();
                                lastXidLogFileOffset = event.getLogPos();
                            }
                        } else if (LogEvent.XID_EVENT == event.getHeader().getType()) {
                            lastXidLogFilename = current.getName();
                            lastXidLogFileOffset = event.getLogPos();
                        }
                    }
                } while (event != null);
            }
            if (needContinue) {
                // 读取下一个
                // 关闭上一个文件
                fetcher.close();
                File nextFile = binlogs.getBefore(current);
                if (nextFile == null) {
                    break;
                }
                current = nextFile;
                fetcher.open(current);
                context.setLogPosition(new LogPosition(current.getName()));
            } else {
                // 跳出
                break;
            }
        }
    } finally {
        if (fetcher != null) {
            fetcher.close();
        }
    }
    dump(binlogFilename, binlogFileOffset, func);
}

Example 3 with LogDecoder

use of com.taobao.tddl.dbsync.binlog.LogDecoder in project canal by alibaba.

the class MysqlConnection method dump.

public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendBinlogDump(binlogfilename, binlogPosition);
    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    fetcher.start(connector.getChannel());
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();
    context.setLogPosition(new LogPosition(binlogfilename));
    context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
    while (fetcher.fetch()) {
        LogEvent event = null;
        event = decoder.decode(fetcher, context);
        if (event == null) {
            throw new CanalParseException("parse failed");
        }
        if (!func.sink(event)) {
            break;
        }
    }
}

Example 4 with LogDecoder

use of com.taobao.tddl.dbsync.binlog.LogDecoder in project canal by alibaba.

the class MysqlConnection method seek.

/**
     * 加速主备切换时的查找速度，做一些特殊优化，比如只解析事务头或者尾
     */
public void seek(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendBinlogDump(binlogfilename, binlogPosition);
    DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize());
    fetcher.start(connector.getChannel());
    LogDecoder decoder = new LogDecoder();
    decoder.handle(LogEvent.ROTATE_EVENT);
    decoder.handle(LogEvent.FORMAT_DESCRIPTION_EVENT);
    decoder.handle(LogEvent.QUERY_EVENT);
    decoder.handle(LogEvent.XID_EVENT);
    LogContext context = new LogContext();
    context.setLogPosition(new LogPosition(binlogfilename));
    context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
    while (fetcher.fetch()) {
        LogEvent event = null;
        event = decoder.decode(fetcher, context);
        if (event == null) {
            throw new CanalParseException("parse failed");
        }
        if (!func.sink(event)) {
            break;
        }
    }
}

Example 5 with LogDecoder

use of com.taobao.tddl.dbsync.binlog.LogDecoder in project canal by alibaba.

the class LocalBinLogConnection method dump.

public void dump(String binlogfilename, Long binlogPosition, SinkFunction func) throws IOException {
    File current = new File(directory, binlogfilename);
    FileLogFetcher fetcher = new FileLogFetcher(bufferSize);
    LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
    LogContext context = new LogContext();
    try {
        fetcher.open(current, binlogPosition);
        context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
        while (running) {
            boolean needContinue = true;
            LogEvent event = null;
            L: while (fetcher.fetch()) {
                do {
                    if (event == null) {
                        event = new RotateLogEvent(context.getLogPosition().getFileName(), context.getLogPosition().getPosition());
                    } else {
                        event = decoder.decode(fetcher, context);
                    }
                    if (event != null && !func.sink(event)) {
                        needContinue = false;
                        break L;
                    }
                } while (event != null);
            }
            if (needContinue) {
                // 读取下一个
                // 关闭上一个文件
                fetcher.close();
                File nextFile;
                if (needWait) {
                    nextFile = binlogs.waitForNextFile(current);
                } else {
                    nextFile = binlogs.getNextFile(current);
                }
                if (nextFile == null) {
                    break;
                }
                current = nextFile;
                fetcher.open(current);
                context.setLogPosition(new LogPosition(nextFile.getName()));
            } else {
                // 跳出
                break;
            }
        }
    } catch (InterruptedException e) {
        logger.warn("LocalBinLogConnection dump interrupted");
    } finally {
        if (fetcher != null) {
            fetcher.close();
        }
    }
}