Examples with LogPosition com.alibaba.otter.canal.protocol.position.LogPosition used on opensource projects

Example 26 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MysqlEventParser method findAsPerTimestampInSpecificLogFile.

/**
 * 根据给定的时间戳，在指定的binlog中找到最接近于该时间戳(必须是小于时间戳)的一个事务起始位置。
 * 针对最后一个binlog会给定endPosition，避免无尽的查询
 */
private EntryPosition findAsPerTimestampInSpecificLogFile(MysqlConnection mysqlConnection, final Long startTimestamp, final EntryPosition endPosition, final String searchBinlogFile, final Boolean justForPositionTimestamp) {
    final LogPosition logPosition = new LogPosition();
    try {
        mysqlConnection.reconnect();
        // 开始遍历文件
        mysqlConnection.seek(searchBinlogFile, 4L, endPosition.getGtid(), new SinkFunction<LogEvent>() {

            private LogPosition lastPosition;

            public boolean sink(LogEvent event) {
                EntryPosition entryPosition = null;
                try {
                    CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true);
                    if (justForPositionTimestamp && logPosition.getPostion() == null && event.getWhen() > 0) {
                        // 初始位点
                        entryPosition = new EntryPosition(searchBinlogFile, event.getLogPos() - event.getEventLen(), event.getWhen() * 1000, event.getServerId());
                        entryPosition.setGtid(event.getHeader().getGtidSetStr());
                        logPosition.setPostion(entryPosition);
                    }
                    // 直接用event的位点来处理,解决一个binlog文件里没有任何事件导致死循环无法退出的问题
                    String logfilename = event.getHeader().getLogFileName();
                    // 记录的是binlog end offest,
                    // 因为与其对比的offest是show master status里的end offest
                    Long logfileoffset = event.getHeader().getLogPos();
                    Long logposTimestamp = event.getHeader().getWhen() * 1000;
                    Long serverId = event.getHeader().getServerId();
                    // 如果最小的一条记录都不满足条件，可直接退出
                    if (logposTimestamp >= startTimestamp) {
                        return false;
                    }
                    if (StringUtils.equals(endPosition.getJournalName(), logfilename) && endPosition.getPosition() <= logfileoffset) {
                        return false;
                    }
                    if (entry == null) {
                        return true;
                    }
                    // data.length，代表该事务的下一条offest，避免多余的事务重复
                    if (CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
                        entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId);
                        if (logger.isDebugEnabled()) {
                            logger.debug("set {} to be pending start position before finding another proper one...", entryPosition);
                        }
                        logPosition.setPostion(entryPosition);
                        entryPosition.setGtid(entry.getHeader().getGtid());
                    } else if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) {
                        // 当前事务开始位点
                        entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp, serverId);
                        if (logger.isDebugEnabled()) {
                            logger.debug("set {} to be pending start position before finding another proper one...", entryPosition);
                        }
                        entryPosition.setGtid(entry.getHeader().getGtid());
                        logPosition.setPostion(entryPosition);
                    }
                    lastPosition = buildLastPosition(entry);
                } catch (Throwable e) {
                    processSinkError(e, lastPosition, searchBinlogFile, 4L);
                }
                return running;
            }
        });
    } catch (IOException e) {
        logger.error("ERROR ## findAsPerTimestampInSpecificLogFile has an error", e);
    }
    if (logPosition.getPostion() != null) {
        return logPosition.getPostion();
    } else {
        return null;
    }
}

Example 27 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MysqlEventParser method findStartPosition.

protected EntryPosition findStartPosition(ErosaConnection connection) throws IOException {
    if (isGTIDMode()) {
        // GTID模式下，CanalLogPositionManager里取最后的gtid，没有则取instanc配置中的
        LogPosition logPosition = getLogPositionManager().getLatestIndexBy(destination);
        if (logPosition != null) {
            // 如果以前是非GTID模式，后来调整为了GTID模式，那么为了保持兼容，需要判断gtid是否为空
            if (StringUtils.isNotEmpty(logPosition.getPostion().getGtid())) {
                return logPosition.getPostion();
            }
        } else {
            if (masterPosition != null && StringUtils.isNotEmpty(masterPosition.getGtid())) {
                return masterPosition;
            }
        }
    }
    EntryPosition startPosition = findStartPositionInternal(connection);
    if (needTransactionPosition.get()) {
        logger.warn("prepare to find last position : {}", startPosition.toString());
        Long preTransactionStartPosition = findTransactionBeginPosition(connection, startPosition);
        if (!preTransactionStartPosition.equals(startPosition.getPosition())) {
            logger.warn("find new start Transaction Position , old : {} , new : {}", startPosition.getPosition(), preTransactionStartPosition);
            startPosition.setPosition(preTransactionStartPosition);
        }
        needTransactionPosition.compareAndSet(true, false);
    }
    return startPosition;
}

Example 28 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MysqlEventParser method findTransactionBeginPosition.

// 根据想要的position，可能这个position对应的记录为rowdata，需要找到事务头，避免丢数据
// 主要考虑一个事务执行时间可能会几秒种，如果仅仅按照timestamp相同，则可能会丢失事务的前半部分数据
private Long findTransactionBeginPosition(ErosaConnection mysqlConnection, final EntryPosition entryPosition) throws IOException {
    // 针对开始的第一条为非Begin记录，需要从该binlog扫描
    final java.util.concurrent.atomic.AtomicLong preTransactionStartPosition = new java.util.concurrent.atomic.AtomicLong(0L);
    mysqlConnection.reconnect();
    mysqlConnection.seek(entryPosition.getJournalName(), 4L, entryPosition.getGtid(), new SinkFunction<LogEvent>() {

        private LogPosition lastPosition;

        public boolean sink(LogEvent event) {
            try {
                CanalEntry.Entry entry = parseAndProfilingIfNecessary(event, true);
                if (entry == null) {
                    return true;
                }
                // 记录一下transaction begin position
                if (entry.getEntryType() == CanalEntry.EntryType.TRANSACTIONBEGIN && entry.getHeader().getLogfileOffset() < entryPosition.getPosition()) {
                    preTransactionStartPosition.set(entry.getHeader().getLogfileOffset());
                }
                if (entry.getHeader().getLogfileOffset() >= entryPosition.getPosition()) {
                    // 退出
                    return false;
                }
                lastPosition = buildLastPosition(entry);
            } catch (Exception e) {
                processSinkError(e, lastPosition, entryPosition.getJournalName(), entryPosition.getPosition());
                return false;
            }
            return running;
        }
    });
    // 判断一下找到的最接近position的事务头的位置
    if (preTransactionStartPosition.get() > entryPosition.getPosition()) {
        logger.error("preTransactionEndPosition greater than startPosition from zk or localconf, maybe lost data");
        throw new CanalParseException("preTransactionStartPosition greater than startPosition from zk or localconf, maybe lost data");
    }
    return preTransactionStartPosition.get();
}

Example 29 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MixedLogPositionManager method getLatestIndexBy.

@Override
public LogPosition getLatestIndexBy(String destination) {
    LogPosition logPosition = memoryLogPositionManager.getLatestIndexBy(destination);
    if (logPosition != null) {
        return logPosition;
    }
    logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
    // 这里保持和重构前的逻辑一致,重新添加到Memory中
    if (logPosition != null) {
        memoryLogPositionManager.persistLogPosition(destination, logPosition);
    }
    return logPosition;
}

Example 30 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MysqlBinlogDumpPerformanceTest method main.

public static void main(String[] args) {
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000007", 89796293L, 100L);
    controller.setConnectionCharset("UTF-8");
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setFilterQueryDml(true);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("100.81.154.142", 3306), "canal", "canal"));
    controller.setMasterPosition(startPosition);
    controller.setEnableTsdb(false);
    controller.setDestination("example");
    controller.setTsdbSpringXml("classpath:spring/tsdb/h2-tsdb.xml");
    // controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
    // controller.setEventBlackFilter(new
    // AviaterRegexFilter("canal_tsdb\\..*"));
    controller.setParallel(true);
    controller.setParallelBufferSize(256);
    controller.setParallelThreadSize(16);
    controller.setIsGTIDMode(false);
    final AtomicLong sum = new AtomicLong(0);
    final AtomicLong last = new AtomicLong(0);
    final AtomicLong start = new AtomicLong(System.currentTimeMillis());
    final AtomicLong end = new AtomicLong(0);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {

        public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            sum.addAndGet(entrys.size());
            long current = sum.get();
            if (current - last.get() >= 100000) {
                end.set(System.currentTimeMillis());
                long tps = ((current - last.get()) * 1000) / (end.get() - start.get());
                System.out.println(" total : " + sum + " , cost : " + (end.get() - start.get()) + " , tps : " + tps);
                last.set(current);
                start.set(end.get());
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000 * 1000L);
    } catch (InterruptedException e) {
    }
    controller.stop();
}