Examples with EntryPosition com.alibaba.otter.canal.protocol.position.EntryPosition used on opensource projects

Example 6 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class LocalBinlogEventParserTest method test_no_position.

@Test
public void test_no_position() throws InterruptedException {
    final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 1000);
    final EntryPosition defaultPosition = buildPosition("mysql-bin.000002", null, new Date().getTime() + 1000 * 1000L);
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final LocalBinlogEventParser controller = new LocalBinlogEventParser();
    controller.setMasterPosition(defaultPosition);
    controller.setMasterInfo(buildAuthentication());
    controller.setDirectory(directory);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (Entry entry : entrys) {
                entryCount.incrementAndGet();
                String logfilename = entry.getHeader().getLogfileName();
                long logfileoffset = entry.getHeader().getLogfileOffset();
                long executeTime = entry.getHeader().getExecuteTime();
                entryPosition.setJournalName(logfilename);
                entryPosition.setPosition(logfileoffset);
                entryPosition.setTimestamp(executeTime);
                break;
            }
            controller.stop();
            timeoutChecker.stop();
            timeoutChecker.touch();
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractCanalLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    // assertEquals(entryPosition.getJournalName(), "mysql-bin.000002");
    Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}

Example 7 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlEventParser method findByStartTimeStamp.

// 根据时间查找binlog位置
private EntryPosition findByStartTimeStamp(MysqlConnection mysqlConnection, Long startTimestamp) {
    EntryPosition endPosition = findEndPosition(mysqlConnection);
    EntryPosition startPosition = findStartPosition(mysqlConnection);
    String maxBinlogFileName = endPosition.getJournalName();
    String minBinlogFileName = startPosition.getJournalName();
    logger.info("show master status to set search end condition:{} ", endPosition);
    String startSearchBinlogFile = endPosition.getJournalName();
    boolean shouldBreak = false;
    while (running && !shouldBreak) {
        try {
            EntryPosition entryPosition = findAsPerTimestampInSpecificLogFile(mysqlConnection, startTimestamp, endPosition, startSearchBinlogFile);
            if (entryPosition == null) {
                if (StringUtils.equalsIgnoreCase(minBinlogFileName, startSearchBinlogFile)) {
                    // 已经找到最早的一个binlog，没必要往前找了
                    shouldBreak = true;
                    logger.warn("Didn't find the corresponding binlog files from {} to {}", minBinlogFileName, maxBinlogFileName);
                } else {
                    // 继续往前找
                    int binlogSeqNum = Integer.parseInt(startSearchBinlogFile.substring(startSearchBinlogFile.indexOf(".") + 1));
                    if (binlogSeqNum <= 1) {
                        logger.warn("Didn't find the corresponding binlog files");
                        shouldBreak = true;
                    } else {
                        int nextBinlogSeqNum = binlogSeqNum - 1;
                        String binlogFileNamePrefix = startSearchBinlogFile.substring(0, startSearchBinlogFile.indexOf(".") + 1);
                        String binlogFileNameSuffix = String.format("%06d", nextBinlogSeqNum);
                        startSearchBinlogFile = binlogFileNamePrefix + binlogFileNameSuffix;
                    }
                }
            } else {
                logger.info("found and return:{} in findByStartTimeStamp operation.", entryPosition);
                return entryPosition;
            }
        } catch (Exception e) {
            logger.warn("the binlogfile:{} doesn't exist, to continue to search the next binlogfile , caused by {}", startSearchBinlogFile, ExceptionUtils.getFullStackTrace(e));
            int binlogSeqNum = Integer.parseInt(startSearchBinlogFile.substring(startSearchBinlogFile.indexOf(".") + 1));
            if (binlogSeqNum <= 1) {
                logger.warn("Didn't find the corresponding binlog files");
                shouldBreak = true;
            } else {
                int nextBinlogSeqNum = binlogSeqNum - 1;
                String binlogFileNamePrefix = startSearchBinlogFile.substring(0, startSearchBinlogFile.indexOf(".") + 1);
                String binlogFileNameSuffix = String.format("%06d", nextBinlogSeqNum);
                startSearchBinlogFile = binlogFileNamePrefix + binlogFileNameSuffix;
            }
        }
    }
    // 找不到
    return null;
}

Example 8 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class AbstractMetaManagerTest method buildRange.

private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}

Example 9 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlEventParser method findAsPerTimestampInSpecificLogFile.

/**
     * 根据给定的时间戳，在指定的binlog中找到最接近于该时间戳(必须是小于时间戳)的一个事务起始位置。
     * 针对最后一个binlog会给定endPosition，避免无尽的查询
     */
private EntryPosition findAsPerTimestampInSpecificLogFile(MysqlConnection mysqlConnection, final Long startTimestamp, final EntryPosition endPosition, final String searchBinlogFile) {
    final LogPosition logPosition = new LogPosition();
    try {
        mysqlConnection.reconnect();
        // 开始遍历文件
        mysqlConnection.seek(searchBinlogFile, 4L, new SinkFunction<LogEvent>() {

            private LogPosition lastPosition;

            public boolean sink(LogEvent event) {
                EntryPosition entryPosition = null;
                try {
                    CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
                    if (entry == null) {
                        return true;
                    }
                    String logfilename = entry.getHeader().getLogfileName();
                    Long logfileoffset = entry.getHeader().getLogfileOffset();
                    Long logposTimestamp = entry.getHeader().getExecuteTime();
                    if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType()) || CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
                        logger.debug("compare exit condition:{},{},{}, startTimestamp={}...", new Object[] { logfilename, logfileoffset, logposTimestamp, startTimestamp });
                        // 事务头和尾寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出
                        if (logposTimestamp >= startTimestamp) {
                            return false;
                        }
                    }
                    if (StringUtils.equals(endPosition.getJournalName(), logfilename) && endPosition.getPosition() <= (logfileoffset + event.getEventLen())) {
                        return false;
                    }
                    // data.length，代表该事务的下一条offest，避免多余的事务重复
                    if (CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
                        entryPosition = new EntryPosition(logfilename, logfileoffset + event.getEventLen(), logposTimestamp);
                        logger.debug("set {} to be pending start position before finding another proper one...", entryPosition);
                        logPosition.setPostion(entryPosition);
                    } else if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) {
                        // 当前事务开始位点
                        entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp);
                        logger.debug("set {} to be pending start position before finding another proper one...", entryPosition);
                        logPosition.setPostion(entryPosition);
                    }
                    lastPosition = buildLastPosition(entry);
                } catch (Throwable e) {
                    processSinkError(e, lastPosition, searchBinlogFile, 4L);
                }
                return running;
            }
        });
    } catch (IOException e) {
        logger.error("ERROR ## findAsPerTimestampInSpecificLogFile has an error", e);
    }
    if (logPosition.getPostion() != null) {
        return logPosition.getPostion();
    } else {
        return null;
    }
}

Example 10 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlEventParserTest method test_position.

@Test
public void test_position() throws InterruptedException {
    final TimeoutChecker timeoutChecker = new TimeoutChecker();
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(true);
    controller.setDetectingSQL(DETECTING_SQL);
    controller.setMasterPosition(defaultPosition);
    controller.setMasterInfo(buildAuthentication());
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        @Override
        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() != EntryType.HEARTBEAT) {
                    entryCount.incrementAndGet();
                    String logfilename = entry.getHeader().getLogfileName();
                    long logfileoffset = entry.getHeader().getLogfileOffset();
                    long executeTime = entry.getHeader().getExecuteTime();
                    entryPosition.setJournalName(logfilename);
                    entryPosition.setPosition(logfileoffset);
                    entryPosition.setTimestamp(executeTime);
                    break;
                }
            }
            if (entryCount.get() > 0) {
                controller.stop();
                timeoutChecker.stop();
                timeoutChecker.touch();
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractCanalLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    Assert.assertEquals(entryPosition, defaultPosition);
}