Examples with CanalParseException com.alibaba.otter.canal.parse.exception.CanalParseException used on opensource projects

Example 1 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class GroupEventPaserTest method buildEventParser.

private MysqlEventParser buildEventParser(int slaveId) {
    MysqlEventParser mysqlEventPaser = new MysqlEventParser();
    EntryPosition defaultPosition = buildPosition("mysql-bin.000001", 6163L, 1322803601000L);
    mysqlEventPaser.setDestination("group-" + slaveId);
    mysqlEventPaser.setSlaveId(slaveId);
    mysqlEventPaser.setDetectingEnable(false);
    mysqlEventPaser.setDetectingSQL(DETECTING_SQL);
    mysqlEventPaser.setMasterInfo(buildAuthentication());
    mysqlEventPaser.setMasterPosition(defaultPosition);
    mysqlEventPaser.setBinlogParser(buildParser(buildAuthentication()));
    mysqlEventPaser.setEventSink(new EntryEventSink());
    mysqlEventPaser.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    return mysqlEventPaser;
}

Example 2 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class LocalBinlogDumpTest method testSimple.

@Test
public void testSimple() {
    String directory = "/Users/wanshao/projects/canal/parse/src/test/resources/binlog/tsdb";
    final LocalBinlogEventParser controller = new LocalBinlogEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 123L);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal"));
    controller.setConnectionCharsetStd(Charset.forName("UTF-8"));
    controller.setDirectory(directory);
    controller.setMasterPosition(startPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
                    continue;
                }
                if (entry.getEntryType() == EntryType.ROWDATA) {
                    RowChange rowChange = null;
                    try {
                        rowChange = RowChange.parseFrom(entry.getStoreValue());
                    } catch (Exception e) {
                        throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                    }
                    EventType eventType = rowChange.getEventType();
                    System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                    for (RowData rowData : rowChange.getRowDatasList()) {
                        if (eventType == EventType.DELETE) {
                            print(rowData.getBeforeColumnsList());
                        } else if (eventType == EventType.INSERT) {
                            print(rowData.getAfterColumnsList());
                        } else {
                            System.out.println("-------> before");
                            print(rowData.getBeforeColumnsList());
                            System.out.println("-------> after");
                            print(rowData.getAfterColumnsList());
                        }
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}

Example 3 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlEventParserTest method test_position.

@Test
public void test_position() throws InterruptedException {
    final TimeoutChecker timeoutChecker = new TimeoutChecker();
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition defaultPosition = buildPosition("mysql-bin.000003", 4690L, 1505481064000L);
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(true);
    controller.setDetectingSQL(DETECTING_SQL);
    controller.setMasterPosition(defaultPosition);
    controller.setMasterInfo(buildAuthentication());
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        @Override
        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() != EntryType.HEARTBEAT) {
                    entryCount.incrementAndGet();
                    String logfilename = entry.getHeader().getLogfileName();
                    long logfileoffset = entry.getHeader().getLogfileOffset();
                    long executeTime = entry.getHeader().getExecuteTime();
                    entryPosition.setJournalName(logfilename);
                    entryPosition.setPosition(logfileoffset);
                    entryPosition.setTimestamp(executeTime);
                    break;
                }
            }
            if (entryCount.get() > 0) {
                controller.stop();
                timeoutChecker.stop();
                timeoutChecker.touch();
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    Assert.assertEquals(entryPosition, defaultPosition);
}

Example 4 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class LocalBinLogConnection method dump.

@Override
public void dump(String binlogfilename, Long binlogPosition, MultiStageCoprocessor coprocessor) throws IOException {
    File current = new File(directory, binlogfilename);
    if (!current.exists()) {
        throw new CanalParseException("binlog:" + binlogfilename + " is not found");
    }
    try (FileLogFetcher fetcher = new FileLogFetcher(bufferSize)) {
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        fetcher.open(current, binlogPosition);
        context.setLogPosition(new LogPosition(binlogfilename, binlogPosition));
        while (running) {
            boolean needContinue = true;
            LogEvent event = null;
            while (fetcher.fetch()) {
                event = decoder.decode(fetcher, context);
                if (event == null) {
                    continue;
                }
                checkServerId(event);
                if (!coprocessor.publish(event)) {
                    needContinue = false;
                    break;
                }
            }
            // 关闭上一个文件
            fetcher.close();
            parserFinish(binlogfilename);
            if (needContinue) {
                // 读取下一个
                File nextFile;
                if (needWait) {
                    nextFile = binlogs.waitForNextFile(current);
                } else {
                    nextFile = binlogs.getNextFile(current);
                }
                if (nextFile == null) {
                    break;
                }
                current = nextFile;
                fetcher.open(current);
                binlogfilename = nextFile.getName();
            } else {
                // 跳出
                break;
            }
        }
    } catch (InterruptedException e) {
        logger.warn("LocalBinLogConnection dump interrupted");
    }
}

Example 5 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlConnection method dump.

@Override
public void dump(GTIDSet gtidSet, SinkFunction func) throws IOException {
    updateSettings();
    loadBinlogChecksum();
    sendBinlogDumpGTID(gtidSet);
    try (DirectLogFetcher fetcher = new DirectLogFetcher(connector.getReceiveBufferSize())) {
        fetcher.start(connector.getChannel());
        LogDecoder decoder = new LogDecoder(LogEvent.UNKNOWN_EVENT, LogEvent.ENUM_END_EVENT);
        LogContext context = new LogContext();
        context.setFormatDescription(new FormatDescriptionLogEvent(4, binlogChecksum));
        // fix bug: #890 将gtid传输至context中，供decode使用
        context.setGtidSet(gtidSet);
        while (fetcher.fetch()) {
            accumulateReceivedBytes(fetcher.limit());
            LogEvent event = null;
            event = decoder.decode(fetcher, context);
            if (event == null) {
                throw new CanalParseException("parse failed");
            }
            if (!func.sink(event)) {
                break;
            }
        }
    }
}