Examples with CanalParseException com.alibaba.otter.canal.parse.exception.CanalParseException used on opensource projects

Example 36 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class DatabaseTableMeta method dumpTableMeta.

/**
 * 初始化的时候dump一下表结构
 */
private boolean dumpTableMeta(MysqlConnection connection, final CanalEventFilter filter) {
    try {
        ResultSetPacket packet = connection.query("show databases");
        List<String> schemas = new ArrayList<>();
        schemas.addAll(packet.getFieldValues());
        for (String schema : schemas) {
            // filter views
            packet = connection.query("show full tables from `" + schema + "` where Table_type = 'BASE TABLE'");
            List<String> tables = new ArrayList<>();
            for (String table : packet.getFieldValues()) {
                if ("BASE TABLE".equalsIgnoreCase(table)) {
                    continue;
                }
                String fullName = schema + "." + table;
                if (blackFilter == null || !blackFilter.filter(fullName)) {
                    if (filter == null || filter.filter(fullName)) {
                        tables.add(table);
                    }
                }
            }
            if (tables.isEmpty()) {
                continue;
            }
            StringBuilder sql = new StringBuilder();
            for (String table : tables) {
                sql.append("show create table `" + schema + "`.`" + table + "`;");
            }
            List<ResultSetPacket> packets = connection.queryMulti(sql.toString());
            for (ResultSetPacket onePacket : packets) {
                if (onePacket.getFieldValues().size() > 1) {
                    String oneTableCreateSql = onePacket.getFieldValues().get(1);
                    memoryTableMeta.apply(INIT_POSITION, schema, oneTableCreateSql, null);
                }
            }
        }
        return true;
    } catch (IOException e) {
        throw new CanalParseException(e);
    }
}

Example 37 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlBinlogDumpPerformanceTest method main.

public static void main(String[] args) {
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000007", 89796293L, 100L);
    controller.setConnectionCharset("UTF-8");
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setFilterQueryDml(true);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("100.81.154.142", 3306), "canal", "canal"));
    controller.setMasterPosition(startPosition);
    controller.setEnableTsdb(false);
    controller.setDestination("example");
    controller.setTsdbSpringXml("classpath:spring/tsdb/h2-tsdb.xml");
    // controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
    // controller.setEventBlackFilter(new
    // AviaterRegexFilter("canal_tsdb\\..*"));
    controller.setParallel(true);
    controller.setParallelBufferSize(256);
    controller.setParallelThreadSize(16);
    controller.setIsGTIDMode(false);
    final AtomicLong sum = new AtomicLong(0);
    final AtomicLong last = new AtomicLong(0);
    final AtomicLong start = new AtomicLong(System.currentTimeMillis());
    final AtomicLong end = new AtomicLong(0);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {

        public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            sum.addAndGet(entrys.size());
            long current = sum.get();
            if (current - last.get() >= 100000) {
                end.set(System.currentTimeMillis());
                long tps = ((current - last.get()) * 1000) / (end.get() - start.get());
                System.out.println(" total : " + sum + " , cost : " + (end.get() - start.get()) + " , tps : " + tps);
                last.set(current);
                start.set(end.get());
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000 * 1000L);
    } catch (InterruptedException e) {
    }
    controller.stop();
}

Example 38 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class MysqlDumpTest method testSimple.

@Test
public void testSimple() {
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000001", 4L);
    // startPosition.setGtid("f1ceb61a-a5d5-11e7-bdee-107c3dbcf8a7:1-17");
    controller.setConnectionCharsetStd(Charset.forName("UTF-8"));
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"));
    controller.setMasterPosition(startPosition);
    controller.setEnableTsdb(true);
    controller.setDestination("example");
    controller.setTsdbSpringXml("classpath:tsdb/h2-tsdb.xml");
    controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
    controller.setEventBlackFilter(new AviaterRegexFilter("canal_tsdb\\..*"));
    controller.setParallel(true);
    controller.setParallelBufferSize(256);
    controller.setParallelThreadSize(2);
    controller.setIsGTIDMode(false);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND || entry.getEntryType() == EntryType.HEARTBEAT) {
                    continue;
                }
                RowChange rowChange = null;
                try {
                    rowChange = RowChange.parseFrom(entry.getStoreValue());
                } catch (Exception e) {
                    throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                }
                EventType eventType = rowChange.getEventType();
                System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                if (eventType == EventType.QUERY || rowChange.getIsDdl()) {
                    System.out.println(" sql ----> " + rowChange.getSql());
                }
                printXAInfo(rowChange.getPropsList());
                for (RowData rowData : rowChange.getRowDatasList()) {
                    if (eventType == EventType.DELETE) {
                        print(rowData.getBeforeColumnsList());
                    } else if (eventType == EventType.INSERT) {
                        print(rowData.getAfterColumnsList());
                    } else {
                        System.out.println("-------> before");
                        print(rowData.getBeforeColumnsList());
                        System.out.println("-------> after");
                        print(rowData.getAfterColumnsList());
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}

Example 39 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class RdsLocalBinlogDumpTest method testSimple.

@Test
public void testSimple() {
    String directory = "/tmp/rds";
    final RdsLocalBinlogEventParser controller = new RdsLocalBinlogEventParser();
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "root", "hello"));
    controller.setConnectionCharsetStd(Charset.forName("UTF-8"));
    controller.setDirectory(directory);
    controller.setAccesskey("");
    controller.setSecretkey("");
    controller.setInstanceId("");
    controller.setStartTime(1507860498350L);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
                    continue;
                }
                if (entry.getEntryType() == EntryType.ROWDATA) {
                    RowChange rowChange = null;
                    try {
                        rowChange = RowChange.parseFrom(entry.getStoreValue());
                    } catch (Exception e) {
                        throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                    }
                    EventType eventType = rowChange.getEventType();
                    System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                    for (RowData rowData : rowChange.getRowDatasList()) {
                        if (eventType == EventType.DELETE) {
                            print(rowData.getBeforeColumnsList());
                        } else if (eventType == EventType.INSERT) {
                            print(rowData.getAfterColumnsList());
                        } else {
                            System.out.println("-------> before");
                            print(rowData.getBeforeColumnsList());
                            System.out.println("-------> after");
                            print(rowData.getAfterColumnsList());
                        }
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}

Example 40 with CanalParseException

use of com.alibaba.otter.canal.parse.exception.CanalParseException in project canal by alibaba.

the class DirectLogFetcherTest method parseTableMapEvent.

public void parseTableMapEvent(TableMapLogEvent event) {
    try {
        String charsetDbName = new String(event.getDbName().getBytes("ISO-8859-1"), charset.name());
        event.setDbname(charsetDbName);
        String charsetTbName = new String(event.getTableName().getBytes("ISO-8859-1"), charset.name());
        event.setTblname(charsetTbName);
    } catch (UnsupportedEncodingException e) {
        throw new CanalParseException(e);
    }
}