Examples with LogPosition com.alibaba.otter.canal.protocol.position.LogPosition used on opensource projects

Example 1 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MixedLogPositionManager method start.

public void start() {
    super.start();
    Assert.notNull(zooKeeperLogPositionManager);
    if (!zooKeeperLogPositionManager.isStart()) {
        zooKeeperLogPositionManager.start();
    }
    executor = Executors.newFixedThreadPool(1);
    positions = MigrateMap.makeComputingMap(new Function<String, LogPosition>() {

        public LogPosition apply(String destination) {
            LogPosition logPosition = zooKeeperLogPositionManager.getLatestIndexBy(destination);
            if (logPosition == null) {
                return nullPosition;
            } else {
                return logPosition;
            }
        }
    });
}

Example 2 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MysqlEventParser method findAsPerTimestampInSpecificLogFile.

/**
     * 根据给定的时间戳，在指定的binlog中找到最接近于该时间戳(必须是小于时间戳)的一个事务起始位置。
     * 针对最后一个binlog会给定endPosition，避免无尽的查询
     */
private EntryPosition findAsPerTimestampInSpecificLogFile(MysqlConnection mysqlConnection, final Long startTimestamp, final EntryPosition endPosition, final String searchBinlogFile) {
    final LogPosition logPosition = new LogPosition();
    try {
        mysqlConnection.reconnect();
        // 开始遍历文件
        mysqlConnection.seek(searchBinlogFile, 4L, new SinkFunction<LogEvent>() {

            private LogPosition lastPosition;

            public boolean sink(LogEvent event) {
                EntryPosition entryPosition = null;
                try {
                    CanalEntry.Entry entry = parseAndProfilingIfNecessary(event);
                    if (entry == null) {
                        return true;
                    }
                    String logfilename = entry.getHeader().getLogfileName();
                    Long logfileoffset = entry.getHeader().getLogfileOffset();
                    Long logposTimestamp = entry.getHeader().getExecuteTime();
                    if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType()) || CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
                        logger.debug("compare exit condition:{},{},{}, startTimestamp={}...", new Object[] { logfilename, logfileoffset, logposTimestamp, startTimestamp });
                        // 事务头和尾寻找第一条记录时间戳，如果最小的一条记录都不满足条件，可直接退出
                        if (logposTimestamp >= startTimestamp) {
                            return false;
                        }
                    }
                    if (StringUtils.equals(endPosition.getJournalName(), logfilename) && endPosition.getPosition() <= (logfileoffset + event.getEventLen())) {
                        return false;
                    }
                    // data.length，代表该事务的下一条offest，避免多余的事务重复
                    if (CanalEntry.EntryType.TRANSACTIONEND.equals(entry.getEntryType())) {
                        entryPosition = new EntryPosition(logfilename, logfileoffset + event.getEventLen(), logposTimestamp);
                        logger.debug("set {} to be pending start position before finding another proper one...", entryPosition);
                        logPosition.setPostion(entryPosition);
                    } else if (CanalEntry.EntryType.TRANSACTIONBEGIN.equals(entry.getEntryType())) {
                        // 当前事务开始位点
                        entryPosition = new EntryPosition(logfilename, logfileoffset, logposTimestamp);
                        logger.debug("set {} to be pending start position before finding another proper one...", entryPosition);
                        logPosition.setPostion(entryPosition);
                    }
                    lastPosition = buildLastPosition(entry);
                } catch (Throwable e) {
                    processSinkError(e, lastPosition, searchBinlogFile, 4L);
                }
                return running;
            }
        });
    } catch (IOException e) {
        logger.error("ERROR ## findAsPerTimestampInSpecificLogFile has an error", e);
    }
    if (logPosition.getPostion() != null) {
        return logPosition.getPostion();
    } else {
        return null;
    }
}

Example 3 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class GroupEventPaserTest method buildEventParser.

private MysqlEventParser buildEventParser(int slaveId) {
    MysqlEventParser mysqlEventPaser = new MysqlEventParser();
    EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
    mysqlEventPaser.setDestination("group-" + slaveId);
    mysqlEventPaser.setSlaveId(slaveId);
    mysqlEventPaser.setDetectingEnable(false);
    mysqlEventPaser.setDetectingSQL(DETECTING_SQL);
    mysqlEventPaser.setMasterInfo(buildAuthentication());
    mysqlEventPaser.setMasterPosition(defaultPosition);
    mysqlEventPaser.setBinlogParser(buildParser(buildAuthentication()));
    mysqlEventPaser.setEventSink(new EntryEventSink());
    mysqlEventPaser.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    return mysqlEventPaser;
}

Example 4 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class LocalBinlogDumpTest method testSimple.

@Test
public void testSimple() {
    String directory = "/Users/wanshao/projects/canal/parse/src/test/resources/binlog/tsdb";
    final LocalBinlogEventParser controller = new LocalBinlogEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 123L);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal"));
    controller.setConnectionCharsetStd(Charset.forName("UTF-8"));
    controller.setDirectory(directory);
    controller.setMasterPosition(startPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
                    continue;
                }
                if (entry.getEntryType() == EntryType.ROWDATA) {
                    RowChange rowChange = null;
                    try {
                        rowChange = RowChange.parseFrom(entry.getStoreValue());
                    } catch (Exception e) {
                        throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                    }
                    EventType eventType = rowChange.getEventType();
                    System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                    for (RowData rowData : rowChange.getRowDatasList()) {
                        if (eventType == EventType.DELETE) {
                            print(rowData.getBeforeColumnsList());
                        } else if (eventType == EventType.INSERT) {
                            print(rowData.getAfterColumnsList());
                        } else {
                            System.out.println("-------> before");
                            print(rowData.getBeforeColumnsList());
                            System.out.println("-------> after");
                            print(rowData.getAfterColumnsList());
                        }
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}

Example 5 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class RdsBinlogEventParserProxyTest method test_timestamp.

@Test
public void test_timestamp() throws InterruptedException {
    final TimeoutChecker timeoutChecker = new TimeoutChecker(3000 * 1000);
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final RdsBinlogEventParserProxy controller = new RdsBinlogEventParserProxy();
    Calendar calendar = Calendar.getInstance();
    calendar.add(Calendar.HOUR_OF_DAY, -24 * 4);
    final EntryPosition defaultPosition = buildPosition(null, null, calendar.getTimeInMillis());
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setDetectingSQL(DETECTING_SQL);
    controller.setMasterInfo(buildAuthentication());
    controller.setMasterPosition(defaultPosition);
    controller.setInstanceId("");
    controller.setAccesskey("");
    controller.setSecretkey("");
    controller.setDirectory("/tmp/binlog");
    controller.setEventBlackFilter(new AviaterRegexFilter("mysql\\.*"));
    controller.setFilterTableError(true);
    controller.setBatchFileSize(4);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {

        @Override
        public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (CanalEntry.Entry entry : entrys) {
                if (entry.getEntryType() != CanalEntry.EntryType.HEARTBEAT) {
                    entryCount.incrementAndGet();
                    String logfilename = entry.getHeader().getLogfileName();
                    long logfileoffset = entry.getHeader().getLogfileOffset();
                    long executeTime = entry.getHeader().getExecuteTime();
                    entryPosition.setJournalName(logfilename);
                    entryPosition.setPosition(logfileoffset);
                    entryPosition.setTimestamp(executeTime);
                    break;
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        private LogPosition logPosition;

        public void persistLogPosition(String destination, LogPosition logPosition) {
            this.logPosition = logPosition;
        }

        public LogPosition getLatestIndexBy(String destination) {
            return logPosition;
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
    Assert.assertTrue(entryPosition.getPosition() <= 6163L);
    Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}