Examples with LogPosition com.alibaba.otter.canal.protocol.position.LogPosition used on opensource projects

Example 36 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MetaLogPositionManagerTest method testAll.

@Test
public void testAll() {
    MixedMetaManager metaManager = new MixedMetaManager();
    ZooKeeperMetaManager zooKeeperMetaManager = new ZooKeeperMetaManager();
    zooKeeperMetaManager.setZkClientx(zkclientx);
    metaManager.setZooKeeperMetaManager(zooKeeperMetaManager);
    metaManager.start();
    MetaLogPositionManager logPositionManager = new MetaLogPositionManager(metaManager);
    logPositionManager.start();
    // 构建meta信息
    ClientIdentity client1 = new ClientIdentity(destination, (short) 1);
    metaManager.subscribe(client1);
    PositionRange range1 = buildRange(1);
    metaManager.updateCursor(client1, range1.getEnd());
    PositionRange range2 = buildRange(2);
    metaManager.updateCursor(client1, range2.getEnd());
    ClientIdentity client2 = new ClientIdentity(destination, (short) 2);
    metaManager.subscribe(client2);
    PositionRange range3 = buildRange(3);
    metaManager.updateCursor(client2, range3.getEnd());
    PositionRange range4 = buildRange(4);
    metaManager.updateCursor(client2, range4.getEnd());
    LogPosition logPosition = logPositionManager.getLatestIndexBy(destination);
    Assert.assertEquals(range2.getEnd(), logPosition);
    metaManager.stop();
    logPositionManager.stop();
}

Example 37 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class PeriodMixedLogPositionManagerTest method testAll.

@Test
public void testAll() {
    MemoryLogPositionManager memoryLogPositionManager = new MemoryLogPositionManager();
    ZooKeeperLogPositionManager zookeeperLogPositionManager = new ZooKeeperLogPositionManager(zkclientx);
    PeriodMixedLogPositionManager logPositionManager = new PeriodMixedLogPositionManager(memoryLogPositionManager, zookeeperLogPositionManager, 1000L);
    logPositionManager.start();
    LogPosition position2 = doTest(logPositionManager);
    sleep(1500);
    PeriodMixedLogPositionManager logPositionManager2 = new PeriodMixedLogPositionManager(memoryLogPositionManager, zookeeperLogPositionManager, 1000L);
    logPositionManager2.start();
    LogPosition getPosition2 = logPositionManager2.getLatestIndexBy(destination);
    Assert.assertEquals(position2, getPosition2);
    logPositionManager.stop();
    logPositionManager2.stop();
}

Example 38 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class MysqlDumpTest method testSimple.

@Test
public void testSimple() {
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000001", 4L);
    // startPosition.setGtid("f1ceb61a-a5d5-11e7-bdee-107c3dbcf8a7:1-17");
    controller.setConnectionCharsetStd(Charset.forName("UTF-8"));
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"));
    controller.setMasterPosition(startPosition);
    controller.setEnableTsdb(true);
    controller.setDestination("example");
    controller.setTsdbSpringXml("classpath:tsdb/h2-tsdb.xml");
    controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
    controller.setEventBlackFilter(new AviaterRegexFilter("canal_tsdb\\..*"));
    controller.setParallel(true);
    controller.setParallelBufferSize(256);
    controller.setParallelThreadSize(2);
    controller.setIsGTIDMode(false);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND || entry.getEntryType() == EntryType.HEARTBEAT) {
                    continue;
                }
                RowChange rowChange = null;
                try {
                    rowChange = RowChange.parseFrom(entry.getStoreValue());
                } catch (Exception e) {
                    throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                }
                EventType eventType = rowChange.getEventType();
                System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                if (eventType == EventType.QUERY || rowChange.getIsDdl()) {
                    System.out.println(" sql ----> " + rowChange.getSql());
                }
                printXAInfo(rowChange.getPropsList());
                for (RowData rowData : rowChange.getRowDatasList()) {
                    if (eventType == EventType.DELETE) {
                        print(rowData.getBeforeColumnsList());
                    } else if (eventType == EventType.INSERT) {
                        print(rowData.getAfterColumnsList());
                    } else {
                        System.out.println("-------> before");
                        print(rowData.getBeforeColumnsList());
                        System.out.println("-------> after");
                        print(rowData.getAfterColumnsList());
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}

Example 39 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class RdsLocalBinlogDumpTest method testSimple.

@Test
public void testSimple() {
    String directory = "/tmp/rds";
    final RdsLocalBinlogEventParser controller = new RdsLocalBinlogEventParser();
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"));
    controller.setConnectionCharsetStd(Charset.forName("UTF-8"));
    controller.setDirectory(directory);
    controller.setAccesskey("");
    controller.setSecretkey("");
    controller.setInstanceId("");
    controller.setStartTime(1507860498350L);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
                    continue;
                }
                if (entry.getEntryType() == EntryType.ROWDATA) {
                    RowChange rowChange = null;
                    try {
                        rowChange = RowChange.parseFrom(entry.getStoreValue());
                    } catch (Exception e) {
                        throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                    }
                    EventType eventType = rowChange.getEventType();
                    System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                    for (RowData rowData : rowChange.getRowDatasList()) {
                        if (eventType == EventType.DELETE) {
                            print(rowData.getBeforeColumnsList());
                        } else if (eventType == EventType.INSERT) {
                            print(rowData.getAfterColumnsList());
                        } else {
                            System.out.println("-------> before");
                            print(rowData.getBeforeColumnsList());
                            System.out.println("-------> after");
                            print(rowData.getAfterColumnsList());
                        }
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}

Example 40 with LogPosition

use of com.alibaba.otter.canal.protocol.position.LogPosition in project canal by alibaba.

the class CanalEventUtils method createPosition.

/**
 * 根据entry创建对应的Position对象
 */
public static LogPosition createPosition(Event event) {
    EntryPosition position = new EntryPosition();
    position.setJournalName(event.getJournalName());
    position.setPosition(event.getPosition());
    position.setTimestamp(event.getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(event.getServerId());
    // add gtid
    position.setGtid(event.getGtid());
    LogPosition logPosition = new LogPosition();
    logPosition.setPostion(position);
    logPosition.setIdentity(event.getLogIdentity());
    return logPosition;
}