Examples with EntryPosition com.alibaba.otter.canal.protocol.position.EntryPosition used on opensource projects

Example 31 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class DatabaseTableMeta method applyHistoryOnMemory.

private boolean applyHistoryOnMemory(EntryPosition position, EntryPosition rollbackPosition) {
    try {
        List<MetaHistoryDO> metaHistoryDOList = metaHistoryDAO.findByTimestamp(destination, position.getTimestamp(), rollbackPosition.getTimestamp());
        if (metaHistoryDOList == null) {
            return true;
        }
        for (MetaHistoryDO metaHistoryDO : metaHistoryDOList) {
            String binlogFile = metaHistoryDO.getBinlogFile();
            Long binlogOffest = metaHistoryDO.getBinlogOffest();
            String binlogMasterId = metaHistoryDO.getBinlogMasterId();
            Long binlogTimestamp = metaHistoryDO.getBinlogTimestamp();
            String useSchema = metaHistoryDO.getUseSchema();
            String sqlData = metaHistoryDO.getSqlText();
            EntryPosition snapshotPosition = new EntryPosition(binlogFile, binlogOffest == null ? 0L : binlogOffest, binlogTimestamp == null ? 0L : binlogTimestamp, Long.valueOf(binlogMasterId == null ? "-2" : binlogMasterId));
            // 如果是同一秒内,对比一下history的位点，如果比期望的位点要大，忽略之
            if (snapshotPosition.getTimestamp() > rollbackPosition.getTimestamp()) {
                continue;
            } else if (rollbackPosition.getServerId().equals(snapshotPosition.getServerId()) && snapshotPosition.compareTo(rollbackPosition) > 0) {
                continue;
            }
            // 记录到内存
            if (!memoryTableMeta.apply(snapshotPosition, useSchema, sqlData, null)) {
                return false;
            }
        }
        return metaHistoryDOList.size() > 0;
    } catch (Throwable e) {
        throw new CanalParseException("apply failed", e);
    }
}

Example 32 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlBinlogDumpPerformanceTest method main.

public static void main(String[] args) {
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000007", 89796293L, 100L);
    controller.setConnectionCharset("UTF-8");
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setFilterQueryDml(true);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("100.81.154.142", 3306), "canal", "canal"));
    controller.setMasterPosition(startPosition);
    controller.setEnableTsdb(false);
    controller.setDestination("example");
    controller.setTsdbSpringXml("classpath:spring/tsdb/h2-tsdb.xml");
    // controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
    // controller.setEventBlackFilter(new
    // AviaterRegexFilter("canal_tsdb\\..*"));
    controller.setParallel(true);
    controller.setParallelBufferSize(256);
    controller.setParallelThreadSize(16);
    controller.setIsGTIDMode(false);
    final AtomicLong sum = new AtomicLong(0);
    final AtomicLong last = new AtomicLong(0);
    final AtomicLong start = new AtomicLong(System.currentTimeMillis());
    final AtomicLong end = new AtomicLong(0);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {

        public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            sum.addAndGet(entrys.size());
            long current = sum.get();
            if (current - last.get() >= 100000) {
                end.set(System.currentTimeMillis());
                long tps = ((current - last.get()) * 1000) / (end.get() - start.get());
                System.out.println(" total : " + sum + " , cost : " + (end.get() - start.get()) + " , tps : " + tps);
                last.set(current);
                start.set(end.get());
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000 * 1000L);
    } catch (InterruptedException e) {
    }
    controller.stop();
}

Example 33 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class LocalBinlogEventParserTest method test_timestamp.

@Test
public void test_timestamp() throws InterruptedException {
    final TimeoutChecker timeoutChecker = new TimeoutChecker(300 * 1000);
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final EntryPosition defaultPosition = buildPosition("mysql-bin.000003", null, 1505467103000L);
    final LocalBinlogEventParser controller = new LocalBinlogEventParser();
    controller.setMasterPosition(defaultPosition);
    controller.setMasterInfo(buildAuthentication());
    controller.setDirectory(directory);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        @Override
        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (Entry entry : entrys) {
                entryCount.incrementAndGet();
                String logfilename = entry.getHeader().getLogfileName();
                long logfileoffset = entry.getHeader().getLogfileOffset();
                long executeTime = entry.getHeader().getExecuteTime();
                entryPosition.setJournalName(logfilename);
                entryPosition.setPosition(logfileoffset);
                entryPosition.setTimestamp(executeTime);
                break;
            }
            controller.stop();
            timeoutChecker.stop();
            timeoutChecker.touch();
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000003");
    Assert.assertTrue(entryPosition.getPosition() <= 300L);
    Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}

Example 34 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlEventParserTest method test_ha.

@Test
public void test_ha() throws InterruptedException {
    final TimeoutChecker timeoutChecker = new TimeoutChecker(30 * 1000);
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(buildAuthentication());
    controller.setMasterPosition(defaultPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        @Override
        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() != EntryType.HEARTBEAT) {
                    entryCount.incrementAndGet();
                    String logfilename = entry.getHeader().getLogfileName();
                    long logfileoffset = entry.getHeader().getLogfileOffset();
                    long executeTime = entry.getHeader().getExecuteTime();
                    entryPosition.setJournalName(logfilename);
                    entryPosition.setPosition(logfileoffset);
                    entryPosition.setTimestamp(executeTime);
                    break;
                }
            }
            if (entryCount.get() > 0) {
                controller.stop();
                timeoutChecker.stop();
                timeoutChecker.touch();
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        public LogPosition getLatestIndexBy(String destination) {
            LogPosition masterLogPosition = new LogPosition();
            masterLogPosition.setIdentity(new LogIdentity(new InetSocketAddress("127.0.0.1", 3306), 1234L));
            masterLogPosition.setPostion(new EntryPosition(1322803601000L));
            return masterLogPosition;
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
    Assert.assertTrue(entryPosition.getPosition() <= 6163L);
    Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}

Example 35 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlEventParserTest method test_no_position.

@Test
public void test_no_position() throws InterruptedException {
    // 在某个文件下，找不到对应的timestamp数据，会使用106L
    // position进行数据抓取
    final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 60 * 1000);
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", null, new Date().getTime() + 1000 * 1000L);
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(buildAuthentication());
    controller.setMasterPosition(defaultPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        @Override
        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() != EntryType.HEARTBEAT) {
                    entryCount.incrementAndGet();
                    // String logfilename =
                    // entry.getHeader().getLogfileName();
                    // long logfileoffset =
                    // entry.getHeader().getLogfileOffset();
                    long executeTime = entry.getHeader().getExecuteTime();
                    // entryPosition.setJournalName(logfilename);
                    // entryPosition.setPosition(logfileoffset);
                    entryPosition.setTimestamp(executeTime);
                    break;
                }
            }
            if (entryCount.get() > 0) {
                controller.stop();
                timeoutChecker.stop();
                timeoutChecker.touch();
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    // Assert.assertEquals(logfilename, "mysql-bin.000001");
    // Assert.assertEquals(106L, logfileoffset);
    Assert.assertTrue(entryPosition.getTimestamp() < defaultPosition.getTimestamp());
}