Search in sources :

Example 36 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class TableMetaManagerTest method testSimple.

@Test
public void testSimple() throws FileNotFoundException, IOException {
    tableMetaManager.init("test");
    URL url = Thread.currentThread().getContextClassLoader().getResource("dummy.txt");
    File dummyFile = new File(url.getFile());
    File create = new File(dummyFile.getParent() + "/ddl", "create.sql");
    EntryPosition position = new EntryPosition("mysql-bin.001115", 139177334L, 3065927853L, 1501660815000L);
    String createSql = StringUtils.join(IOUtils.readLines(new FileInputStream(create)), "\n");
    tableMetaManager.apply(position, "tddl5_00", createSql, null);
    String alterSql = "alter table `test` add column name varchar(32) after c_varchar";
    position = new EntryPosition("mysql-bin.001115", 139177334L, 3065927854L, 1501660815000L);
    tableMetaManager.apply(position, "tddl5_00", alterSql, null);
}
Also used : EntryPosition(com.alibaba.otter.canal.protocol.position.EntryPosition) File(java.io.File) URL(java.net.URL) FileInputStream(java.io.FileInputStream) Test(org.junit.Test)

Example 37 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MetaLogPositionManagerTest method buildRange.

private PositionRange<LogPosition> buildRange(int number) {
    LogPosition start = new LogPosition();
    start.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    start.setPostion(new EntryPosition("mysql-bin.000000" + number, 106L, new Date().getTime()));
    LogPosition end = new LogPosition();
    end.setIdentity(new LogIdentity(new InetSocketAddress(MYSQL_ADDRESS, 3306), 1234L));
    end.setPostion(new EntryPosition("mysql-bin.000000" + (number + 1), 106L, (new Date().getTime()) + 1000 * 1000L));
    return new PositionRange<>(start, end);
}
Also used : InetSocketAddress(java.net.InetSocketAddress) PositionRange(com.alibaba.otter.canal.protocol.position.PositionRange) LogIdentity(com.alibaba.otter.canal.protocol.position.LogIdentity) EntryPosition(com.alibaba.otter.canal.protocol.position.EntryPosition) Date(java.util.Date) LogPosition(com.alibaba.otter.canal.protocol.position.LogPosition)

Example 38 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class MysqlDumpTest method testSimple.

@Test
public void testSimple() {
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000001", 4L);
    // startPosition.setGtid("f1ceb61a-a5d5-11e7-bdee-107c3dbcf8a7:1-17");
    controller.setConnectionCharsetStd(Charset.forName("UTF-8"));
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"));
    controller.setMasterPosition(startPosition);
    controller.setEnableTsdb(true);
    controller.setDestination("example");
    controller.setTsdbSpringXml("classpath:tsdb/h2-tsdb.xml");
    controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
    controller.setEventBlackFilter(new AviaterRegexFilter("canal_tsdb\\..*"));
    controller.setParallel(true);
    controller.setParallelBufferSize(256);
    controller.setParallelThreadSize(2);
    controller.setIsGTIDMode(false);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND || entry.getEntryType() == EntryType.HEARTBEAT) {
                    continue;
                }
                RowChange rowChange = null;
                try {
                    rowChange = RowChange.parseFrom(entry.getStoreValue());
                } catch (Exception e) {
                    throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                }
                EventType eventType = rowChange.getEventType();
                System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                if (eventType == EventType.QUERY || rowChange.getIsDdl()) {
                    System.out.println(" sql ----> " + rowChange.getSql());
                }
                printXAInfo(rowChange.getPropsList());
                for (RowData rowData : rowChange.getRowDatasList()) {
                    if (eventType == EventType.DELETE) {
                        print(rowData.getBeforeColumnsList());
                    } else if (eventType == EventType.INSERT) {
                        print(rowData.getAfterColumnsList());
                    } else {
                        System.out.println("-------> before");
                        print(rowData.getBeforeColumnsList());
                        System.out.println("-------> after");
                        print(rowData.getAfterColumnsList());
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}
Also used : AviaterRegexFilter(com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter) RowChange(com.alibaba.otter.canal.protocol.CanalEntry.RowChange) EventType(com.alibaba.otter.canal.protocol.CanalEntry.EventType) InetSocketAddress(java.net.InetSocketAddress) AbstractLogPositionManager(com.alibaba.otter.canal.parse.index.AbstractLogPositionManager) AuthenticationInfo(com.alibaba.otter.canal.parse.support.AuthenticationInfo) CanalSinkException(com.alibaba.otter.canal.sink.exception.CanalSinkException) CanalParseException(com.alibaba.otter.canal.parse.exception.CanalParseException) CanalParseException(com.alibaba.otter.canal.parse.exception.CanalParseException) Entry(com.alibaba.otter.canal.protocol.CanalEntry.Entry) RowData(com.alibaba.otter.canal.protocol.CanalEntry.RowData) List(java.util.List) EntryPosition(com.alibaba.otter.canal.protocol.position.EntryPosition) CanalSinkException(com.alibaba.otter.canal.sink.exception.CanalSinkException) LogPosition(com.alibaba.otter.canal.protocol.position.LogPosition) Test(org.junit.Test) AbstractCanalEventSinkTest(com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest)

Example 39 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class CanalEventUtils method createPosition.

/**
 * 根据entry创建对应的Position对象
 */
public static LogPosition createPosition(Event event) {
    EntryPosition position = new EntryPosition();
    position.setJournalName(event.getJournalName());
    position.setPosition(event.getPosition());
    position.setTimestamp(event.getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(event.getServerId());
    // add gtid
    position.setGtid(event.getGtid());
    LogPosition logPosition = new LogPosition();
    logPosition.setPostion(position);
    logPosition.setIdentity(event.getLogIdentity());
    return logPosition;
}
Also used : EntryPosition(com.alibaba.otter.canal.protocol.position.EntryPosition) LogPosition(com.alibaba.otter.canal.protocol.position.LogPosition)

Example 40 with EntryPosition

use of com.alibaba.otter.canal.protocol.position.EntryPosition in project canal by alibaba.

the class AbstractEventParser method buildLastPosition.

protected LogPosition buildLastPosition(CanalEntry.Entry entry) {
    // 初始化一下
    LogPosition logPosition = new LogPosition();
    EntryPosition position = new EntryPosition();
    position.setJournalName(entry.getHeader().getLogfileName());
    position.setPosition(entry.getHeader().getLogfileOffset());
    position.setTimestamp(entry.getHeader().getExecuteTime());
    // add serverId at 2016-06-28
    position.setServerId(entry.getHeader().getServerId());
    // set gtid
    position.setGtid(entry.getHeader().getGtid());
    logPosition.setPostion(position);
    LogIdentity identity = new LogIdentity(runningInfo.getAddress(), -1L);
    logPosition.setIdentity(identity);
    return logPosition;
}
Also used : LogIdentity(com.alibaba.otter.canal.protocol.position.LogIdentity) EntryPosition(com.alibaba.otter.canal.protocol.position.EntryPosition) LogPosition(com.alibaba.otter.canal.protocol.position.LogPosition)

Aggregations

EntryPosition (com.alibaba.otter.canal.protocol.position.EntryPosition)40 LogPosition (com.alibaba.otter.canal.protocol.position.LogPosition)24 CanalParseException (com.alibaba.otter.canal.parse.exception.CanalParseException)16 InetSocketAddress (java.net.InetSocketAddress)14 AbstractLogPositionManager (com.alibaba.otter.canal.parse.index.AbstractLogPositionManager)12 List (java.util.List)12 CanalSinkException (com.alibaba.otter.canal.sink.exception.CanalSinkException)11 Test (org.junit.Test)11 AbstractCanalEventSinkTest (com.alibaba.otter.canal.parse.stub.AbstractCanalEventSinkTest)10 Entry (com.alibaba.otter.canal.protocol.CanalEntry.Entry)10 AtomicLong (java.util.concurrent.atomic.AtomicLong)10 TimeoutChecker (com.alibaba.otter.canal.parse.helper.TimeoutChecker)8 IOException (java.io.IOException)6 Date (java.util.Date)6 CanalEntry (com.alibaba.otter.canal.protocol.CanalEntry)5 LogIdentity (com.alibaba.otter.canal.protocol.position.LogIdentity)5 AuthenticationInfo (com.alibaba.otter.canal.parse.support.AuthenticationInfo)4 AviaterRegexFilter (com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter)3 PositionNotFoundException (com.alibaba.otter.canal.parse.exception.PositionNotFoundException)3 MysqlEventParser (com.alibaba.otter.canal.parse.inbound.mysql.MysqlEventParser)3