Examples with EntryPosition com.alibaba.otter.canal.protocol.position.EntryPosition used on opensource projects

Example 16 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlEventParser method findEndPosition.

/**
     * 查询当前的binlog位置
     */
private EntryPosition findEndPosition(MysqlConnection mysqlConnection) {
    try {
        ResultSetPacket packet = mysqlConnection.query("show master status");
        List<String> fields = packet.getFieldValues();
        if (CollectionUtils.isEmpty(fields)) {
            throw new CanalParseException("command : 'show master status' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
        }
        EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
        return endPosition;
    } catch (IOException e) {
        throw new CanalParseException("command : 'show master status' has an error!", e);
    }
}

Example 17 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlEventParser method findStartPositionInternal.

protected EntryPosition findStartPositionInternal(ErosaConnection connection) {
    MysqlConnection mysqlConnection = (MysqlConnection) connection;
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    if (logPosition == null) {
        // 找不到历史成功记录
        EntryPosition entryPosition = null;
        if (masterInfo != null && mysqlConnection.getConnector().getAddress().equals(masterInfo.getAddress())) {
            entryPosition = masterPosition;
        } else if (standbyInfo != null && mysqlConnection.getConnector().getAddress().equals(standbyInfo.getAddress())) {
            entryPosition = standbyPosition;
        }
        if (entryPosition == null) {
            // 默认从当前最后一个位置进行消费
            entryPosition = findEndPosition(mysqlConnection);
        }
        // 判断一下是否需要按时间订阅
        if (StringUtils.isEmpty(entryPosition.getJournalName())) {
            // 如果没有指定binlogName，尝试按照timestamp进行查找
            if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                logger.warn("prepare to find start position {}:{}:{}", new Object[] { "", "", entryPosition.getTimestamp() });
                return findByStartTimeStamp(mysqlConnection, entryPosition.getTimestamp());
            } else {
                logger.warn("prepare to find start position just show master status");
                // 默认从当前最后一个位置进行消费
                return findEndPosition(mysqlConnection);
            }
        } else {
            if (entryPosition.getPosition() != null && entryPosition.getPosition() > 0L) {
                // 如果指定binlogName + offest，直接返回
                logger.warn("prepare to find start position {}:{}:{}", new Object[] { entryPosition.getJournalName(), entryPosition.getPosition(), "" });
                return entryPosition;
            } else {
                EntryPosition specificLogFilePosition = null;
                if (entryPosition.getTimestamp() != null && entryPosition.getTimestamp() > 0L) {
                    // 如果指定binlogName +
                    // timestamp，但没有指定对应的offest，尝试根据时间找一下offest
                    EntryPosition endPosition = findEndPosition(mysqlConnection);
                    if (endPosition != null) {
                        logger.warn("prepare to find start position {}:{}:{}", new Object[] { entryPosition.getJournalName(), "", entryPosition.getTimestamp() });
                        specificLogFilePosition = findAsPerTimestampInSpecificLogFile(mysqlConnection, entryPosition.getTimestamp(), endPosition, entryPosition.getJournalName());
                    }
                }
                if (specificLogFilePosition == null) {
                    // position不存在，从文件头开始
                    entryPosition.setPosition(BINLOG_START_OFFEST);
                    return entryPosition;
                } else {
                    return specificLogFilePosition;
                }
            }
        }
    } else {
        if (logPosition.getIdentity().getSourceAddress().equals(mysqlConnection.getConnector().getAddress())) {
            if (dumpErrorCountThreshold >= 0 && dumpErrorCount > dumpErrorCountThreshold) {
                // binlog定位位点失败,可能有两个原因:
                // 1. binlog位点被删除
                // 2.vip模式的mysql,发生了主备切换,判断一下serverId是否变化,针对这种模式可以发起一次基于时间戳查找合适的binlog位点
                boolean case2 = (standbyInfo == null || standbyInfo.getAddress() == null) && logPosition.getPostion().getServerId() != null && !logPosition.getPostion().getServerId().equals(findServerId(mysqlConnection));
                if (case2) {
                    long timestamp = logPosition.getPostion().getTimestamp();
                    long newStartTimestamp = timestamp - fallbackIntervalInSeconds * 1000;
                    logger.warn("prepare to find start position by last position {}:{}:{}", new Object[] { "", "", logPosition.getPostion().getTimestamp() });
                    EntryPosition findPosition = findByStartTimeStamp(mysqlConnection, newStartTimestamp);
                    // 重新置为一下
                    dumpErrorCount = 0;
                    return findPosition;
                }
            }
            // 其余情况
            logger.warn("prepare to find start position just last position\n {}", JsonUtils.marshalToString(logPosition));
            return logPosition.getPostion();
        } else {
            // 针对切换的情况，考虑回退时间
            long newStartTimestamp = logPosition.getPostion().getTimestamp() - fallbackIntervalInSeconds * 1000;
            logger.warn("prepare to find start position by switch {}:{}:{}", new Object[] { "", "", logPosition.getPostion().getTimestamp() });
            return findByStartTimeStamp(mysqlConnection, newStartTimestamp);
        }
    }
}

Example 18 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlEventParser method findStartPosition.

protected EntryPosition findStartPosition(ErosaConnection connection) throws IOException {
    EntryPosition startPosition = findStartPositionInternal(connection);
    if (needTransactionPosition.get()) {
        logger.warn("prepare to find last position : {}", startPosition.toString());
        Long preTransactionStartPosition = findTransactionBeginPosition(connection, startPosition);
        if (!preTransactionStartPosition.equals(startPosition.getPosition())) {
            logger.warn("find new start Transaction Position , old : {} , new : {}", startPosition.getPosition(), preTransactionStartPosition);
            startPosition.setPosition(preTransactionStartPosition);
        }
        needTransactionPosition.compareAndSet(true, false);
    }
    return startPosition;
}

Example 19 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class LocalBinlogEventParserTest method test_timestamp.

@Test
public void test_timestamp() throws InterruptedException {
    final TimeoutChecker timeoutChecker = new TimeoutChecker(300 * 1000);
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", null, 1322803601000L);
    final LocalBinlogEventParser controller = new LocalBinlogEventParser();
    controller.setMasterPosition(defaultPosition);
    controller.setMasterInfo(buildAuthentication());
    controller.setDirectory(directory);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        @Override
        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (Entry entry : entrys) {
                entryCount.incrementAndGet();
                String logfilename = entry.getHeader().getLogfileName();
                long logfileoffset = entry.getHeader().getLogfileOffset();
                long executeTime = entry.getHeader().getExecuteTime();
                entryPosition.setJournalName(logfilename);
                entryPosition.setPosition(logfileoffset);
                entryPosition.setTimestamp(executeTime);
                break;
            }
            controller.stop();
            timeoutChecker.stop();
            timeoutChecker.touch();
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractCanalLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
    Assert.assertTrue(entryPosition.getPosition() <= 6163L);
    Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}

Example 20 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlEventParser method findStartPosition.

/**
     * 查询当前的binlog位置
     */
private EntryPosition findStartPosition(MysqlConnection mysqlConnection) {
    try {
        ResultSetPacket packet = mysqlConnection.query("show binlog events limit 1");
        List<String> fields = packet.getFieldValues();
        if (CollectionUtils.isEmpty(fields)) {
            throw new CanalParseException("command : 'show binlog events limit 1' has an error! pls check. you need (at least one of) the SUPER,REPLICATION CLIENT privilege(s) for this operation");
        }
        EntryPosition endPosition = new EntryPosition(fields.get(0), Long.valueOf(fields.get(1)));
        return endPosition;
    } catch (IOException e) {
        throw new CanalParseException("command : 'show binlog events limit 1' has an error!", e);
    }
}