Examples with EntryPosition com.alibaba.otter.canal.protocol.position.EntryPosition used on opensource projects

Example 21 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class AbstractEventParser method buildLastPosition.

protected LogPosition buildLastPosition(CanalEntry.Entry entry) {
    // 初始化一下
    LogPosition logPosition = new LogPosition();
    EntryPosition position = new EntryPosition();
    position.setJournalName(entry.getHeader().getLogfileName());
    position.setPosition(entry.getHeader().getLogfileOffset());
    position.setTimestamp(entry.getHeader().getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(entry.getHeader().getServerId());
    logPosition.setPostion(position);
    LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
    logPosition.setIdentity(identity);
    return logPosition;
}

Example 22 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlDumpTest method testSimple.

@Test
public void testSimple() {
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 4L);
    controller.setConnectionCharset(Charset.forName("UTF-8"));
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "xxxxx", "xxxxx"));
    controller.setMasterPosition(startPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND || entry.getEntryType() == EntryType.HEARTBEAT) {
                    continue;
                }
                RowChange rowChage = null;
                try {
                    rowChage = RowChange.parseFrom(entry.getStoreValue());
                } catch (Exception e) {
                    throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                }
                EventType eventType = rowChage.getEventType();
                System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                if (eventType == EventType.QUERY || rowChage.getIsDdl()) {
                    System.out.println(" sql ----> " + rowChage.getSql());
                }
                for (RowData rowData : rowChage.getRowDatasList()) {
                    if (eventType == EventType.DELETE) {
                        print(rowData.getBeforeColumnsList());
                    } else if (eventType == EventType.INSERT) {
                        print(rowData.getAfterColumnsList());
                    } else {
                        System.out.println("-------> before");
                        print(rowData.getBeforeColumnsList());
                        System.out.println("-------> after");
                        print(rowData.getAfterColumnsList());
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractCanalLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}

Example 23 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlEventParserTest method test_no_position.

@Test
public void test_no_position() throws InterruptedException {
    // 在某个文件下，找不到对应的timestamp数据，会使用106L
    // position进行数据抓取
    final TimeoutChecker timeoutChecker = new TimeoutChecker(3 * 60 * 1000);
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", null, new Date().getTime() + 1000 * 1000L);
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(buildAuthentication());
    controller.setMasterPosition(defaultPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        @Override
        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() != EntryType.HEARTBEAT) {
                    entryCount.incrementAndGet();
                    // String logfilename =
                    // entry.getHeader().getLogfileName();
                    // long logfileoffset =
                    // entry.getHeader().getLogfileOffset();
                    long executeTime = entry.getHeader().getExecuteTime();
                    // entryPosition.setJournalName(logfilename);
                    // entryPosition.setPosition(logfileoffset);
                    entryPosition.setTimestamp(executeTime);
                    break;
                }
            }
            if (entryCount.get() > 0) {
                controller.stop();
                timeoutChecker.stop();
                timeoutChecker.touch();
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractCanalLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    // Assert.assertEquals(logfilename, "mysql-bin.000001");
    // Assert.assertEquals(106L, logfileoffset);
    Assert.assertTrue(entryPosition.getTimestamp() < defaultPosition.getTimestamp());
}

Example 24 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlEventParserTest method test_ha.

@Test
public void test_ha() throws InterruptedException {
    final TimeoutChecker timeoutChecker = new TimeoutChecker(30 * 1000);
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(buildAuthentication());
    controller.setMasterPosition(defaultPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        @Override
        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() != EntryType.HEARTBEAT) {
                    entryCount.incrementAndGet();
                    String logfilename = entry.getHeader().getLogfileName();
                    long logfileoffset = entry.getHeader().getLogfileOffset();
                    long executeTime = entry.getHeader().getExecuteTime();
                    entryPosition.setJournalName(logfilename);
                    entryPosition.setPosition(logfileoffset);
                    entryPosition.setTimestamp(executeTime);
                    break;
                }
            }
            if (entryCount.get() > 0) {
                controller.stop();
                timeoutChecker.stop();
                timeoutChecker.touch();
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractCanalLogPositionManager() {

        public void persistLogPosition(String destination, LogPosition logPosition) {
            System.out.println(logPosition);
        }

        public LogPosition getLatestIndexBy(String destination) {
            LogPosition masterLogPosition = new LogPosition();
            masterLogPosition.setIdentity(new LogIdentity(new InetSocketAddress("127.0.0.1", 3306), 1234L));
            masterLogPosition.setPostion(new EntryPosition(1322803601000L));
            return masterLogPosition;
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
    Assert.assertTrue(entryPosition.getPosition() <= 6163L);
    Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}

Example 25 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MetaLogPositionManagerTest method buildRange.

private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<LogPosition>(start, end);
}