Examples with AuthenticationInfo com.alibaba.otter.canal.parse.support.AuthenticationInfo used on opensource projects

Example 1 with AuthenticationInfo

use of com.alibaba.otter.canal.parse.support.AuthenticationInfo in project otter by alibaba.

the class CanalEmbedSelector method start.

public void start() {
    if (running) {
        return;
    }
    // 获取destination/filter参数
    Pipeline pipeline = configClientService.findPipeline(pipelineId);
    filter = CanalFilterSupport.makeFilterExpression(pipeline);
    destination = pipeline.getParameters().getDestinationName();
    batchSize = pipeline.getParameters().getMainstemBatchsize();
    batchTimeout = pipeline.getParameters().getBatchTimeout();
    ddlSync = pipeline.getParameters().getDdlSync();
    final boolean syncFull = pipeline.getParameters().getSyncMode().isRow() || pipeline.getParameters().isEnableRemedy();
    // 暂时使用skip load代替
    filterTableError = pipeline.getParameters().getSkipSelectException();
    if (pipeline.getParameters().getDumpSelector() != null) {
        dump = pipeline.getParameters().getDumpSelector();
    }
    if (pipeline.getParameters().getDumpSelectorDetail() != null) {
        dumpDetail = pipeline.getParameters().getDumpSelectorDetail();
    }
    canalServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {

        public CanalInstance generate(String destination) {
            Canal canal = canalConfigClient.findCanal(destination);
            final OtterAlarmHandler otterAlarmHandler = new OtterAlarmHandler();
            otterAlarmHandler.setPipelineId(pipelineId);
            // 注入一下spring资源
            OtterContextLocator.autowire(otterAlarmHandler);
            // 设置下slaveId，保证多个piplineId下重复引用时不重复
            // 默认基数
            long slaveId = 10000;
            if (canal.getCanalParameter().getSlaveId() != null) {
                slaveId = canal.getCanalParameter().getSlaveId();
            }
            canal.getCanalParameter().setSlaveId(slaveId + pipelineId);
            canal.getCanalParameter().setDdlIsolation(ddlSync);
            canal.getCanalParameter().setFilterTableError(filterTableError);
            canal.getCanalParameter().setMemoryStorageRawEntry(false);
            CanalInstanceWithManager instance = new CanalInstanceWithManager(canal, filter) {

                protected CanalHAController initHaController() {
                    HAMode haMode = parameters.getHaMode();
                    if (haMode.isMedia()) {
                        return new MediaHAController(parameters.getMediaGroup(), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName());
                    } else {
                        return super.initHaController();
                    }
                }

                protected void startEventParserInternal(CanalEventParser parser, boolean isGroup) {
                    super.startEventParserInternal(parser, isGroup);
                    if (eventParser instanceof MysqlEventParser) {
                        // 设置支持的类型
                        ((MysqlEventParser) eventParser).setSupportBinlogFormats("ROW");
                        if (syncFull) {
                            ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL");
                        } else {
                            ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL,MINIMAL");
                        }
                        MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
                        // otter先使用简单的模式
                        mysqlEventParser.setParallel(false);
                        CanalHAController haController = mysqlEventParser.getHaController();
                        if (haController instanceof MediaHAController) {
                            if (isGroup) {
                                throw new CanalException("not support group database use media HA");
                            }
                            ((MediaHAController) haController).setCanalHASwitchable(mysqlEventParser);
                        }
                        if (!haController.isStart()) {
                            haController.start();
                        }
                        // 基于media的Ha，直接从tddl中获取数据库信息
                        if (haController instanceof MediaHAController) {
                            AuthenticationInfo authenticationInfo = ((MediaHAController) haController).getAvailableAuthenticationInfo();
                            ((MysqlEventParser) eventParser).setMasterInfo(authenticationInfo);
                        }
                    }
                }
            };
            instance.setAlarmHandler(otterAlarmHandler);
            CanalEventSink eventSink = instance.getEventSink();
            if (eventSink instanceof AbstractCanalEventSink) {
                handler = new OtterDownStreamHandler();
                handler.setPipelineId(pipelineId);
                handler.setDetectingIntervalInSeconds(canal.getCanalParameter().getDetectingIntervalInSeconds());
                // 注入一下spring资源
                OtterContextLocator.autowire(handler);
                // 添加到开头
                ((AbstractCanalEventSink) eventSink).addHandler(handler, 0);
                handler.start();
            }
            return instance;
        }
    });
    canalServer.start();
    canalServer.start(destination);
    this.clientIdentity = new ClientIdentity(destination, pipeline.getParameters().getMainstemClientId(), filter);
    // 发起一次订阅
    canalServer.subscribe(clientIdentity);
    running = true;
}

Example 2 with AuthenticationInfo

use of com.alibaba.otter.canal.parse.support.AuthenticationInfo in project canal by alibaba.

the class LocalBinlogDumpTest method testSimple.

@Test
public void testSimple() {
    String directory = "/Users/wanshao/projects/canal/parse/src/test/resources/binlog/tsdb";
    final LocalBinlogEventParser controller = new LocalBinlogEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000003", 123L);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("127.0.0.1", 3306), "canal", "canal"));
    controller.setConnectionCharsetStd(Charset.forName("UTF-8"));
    controller.setDirectory(directory);
    controller.setMasterPosition(startPosition);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<Entry>>() {

        public boolean sink(List<Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            for (Entry entry : entrys) {
                if (entry.getEntryType() == EntryType.TRANSACTIONBEGIN || entry.getEntryType() == EntryType.TRANSACTIONEND) {
                    continue;
                }
                if (entry.getEntryType() == EntryType.ROWDATA) {
                    RowChange rowChange = null;
                    try {
                        rowChange = RowChange.parseFrom(entry.getStoreValue());
                    } catch (Exception e) {
                        throw new RuntimeException("ERROR ## parser of eromanga-event has an error , data:" + entry.toString(), e);
                    }
                    EventType eventType = rowChange.getEventType();
                    System.out.println(String.format("================> binlog[%s:%s] , name[%s,%s] , eventType : %s", entry.getHeader().getLogfileName(), entry.getHeader().getLogfileOffset(), entry.getHeader().getSchemaName(), entry.getHeader().getTableName(), eventType));
                    for (RowData rowData : rowChange.getRowDatasList()) {
                        if (eventType == EventType.DELETE) {
                            print(rowData.getBeforeColumnsList());
                        } else if (eventType == EventType.INSERT) {
                            print(rowData.getAfterColumnsList());
                        } else {
                            System.out.println("-------> before");
                            print(rowData.getBeforeColumnsList());
                            System.out.println("-------> after");
                            print(rowData.getAfterColumnsList());
                        }
                    }
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
            System.out.println(logPosition);
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000L);
    } catch (InterruptedException e) {
        Assert.fail(e.getMessage());
    }
    controller.stop();
}

Example 3 with AuthenticationInfo

use of com.alibaba.otter.canal.parse.support.AuthenticationInfo in project canal by alibaba.

the class CanalInstanceWithManager method doInitEventParser.

private CanalEventParser doInitEventParser(SourcingType type, List<InetSocketAddress> dbAddresses) {
    CanalEventParser eventParser;
    if (type.isMysql()) {
        MysqlEventParser mysqlEventParser = null;
        if (StringUtils.isNotEmpty(parameters.getRdsAccesskey()) && StringUtils.isNotEmpty(parameters.getRdsSecretkey()) && StringUtils.isNotEmpty(parameters.getRdsInstanceId())) {
            mysqlEventParser = new RdsBinlogEventParserProxy();
            ((RdsBinlogEventParserProxy) mysqlEventParser).setAccesskey(parameters.getRdsAccesskey());
            ((RdsBinlogEventParserProxy) mysqlEventParser).setSecretkey(parameters.getRdsSecretkey());
            ((RdsBinlogEventParserProxy) mysqlEventParser).setInstanceId(parameters.getRdsInstanceId());
        } else {
            mysqlEventParser = new MysqlEventParser();
        }
        mysqlEventParser.setDestination(destination);
        // 编码参数
        mysqlEventParser.setConnectionCharset(parameters.getConnectionCharset());
        mysqlEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
        // 网络相关参数
        mysqlEventParser.setDefaultConnectionTimeoutInSeconds(parameters.getDefaultConnectionTimeoutInSeconds());
        mysqlEventParser.setSendBufferSize(parameters.getSendBufferSize());
        mysqlEventParser.setReceiveBufferSize(parameters.getReceiveBufferSize());
        // 心跳检查参数
        mysqlEventParser.setDetectingEnable(parameters.getDetectingEnable());
        mysqlEventParser.setDetectingSQL(parameters.getDetectingSQL());
        mysqlEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
        // 数据库信息参数
        mysqlEventParser.setSlaveId(parameters.getSlaveId());
        if (!CollectionUtils.isEmpty(dbAddresses)) {
            mysqlEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName()));
            if (dbAddresses.size() > 1) {
                mysqlEventParser.setStandbyInfo(new AuthenticationInfo(dbAddresses.get(1), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName()));
            }
        }
        if (!CollectionUtils.isEmpty(parameters.getPositions())) {
            EntryPosition masterPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(0), EntryPosition.class);
            // binlog位置参数
            mysqlEventParser.setMasterPosition(masterPosition);
            if (parameters.getPositions().size() > 1) {
                EntryPosition standbyPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(1), EntryPosition.class);
                mysqlEventParser.setStandbyPosition(standbyPosition);
            }
        }
        mysqlEventParser.setFallbackIntervalInSeconds(parameters.getFallbackIntervalInSeconds());
        mysqlEventParser.setProfilingEnabled(false);
        mysqlEventParser.setFilterTableError(parameters.getFilterTableError());
        mysqlEventParser.setParallel(parameters.getParallel());
        mysqlEventParser.setIsGTIDMode(BooleanUtils.toBoolean(parameters.getGtidEnable()));
        // tsdb
        if (parameters.getTsdbSnapshotInterval() != null) {
            mysqlEventParser.setTsdbSnapshotInterval(parameters.getTsdbSnapshotInterval());
        }
        if (parameters.getTsdbSnapshotExpire() != null) {
            mysqlEventParser.setTsdbSnapshotExpire(parameters.getTsdbSnapshotExpire());
        }
        boolean tsdbEnable = BooleanUtils.toBoolean(parameters.getTsdbEnable());
        if (tsdbEnable) {
            mysqlEventParser.setTableMetaTSDBFactory(new DefaultTableMetaTSDBFactory() {

                @Override
                public void destory(String destination) {
                    TableMetaTSDBBuilder.destory(destination);
                }

                @Override
                public TableMetaTSDB build(String destination, String springXml) {
                    try {
                        System.setProperty("canal.instance.tsdb.url", parameters.getTsdbJdbcUrl());
                        System.setProperty("canal.instance.tsdb.dbUsername", parameters.getTsdbJdbcUserName());
                        System.setProperty("canal.instance.tsdb.dbPassword", parameters.getTsdbJdbcPassword());
                        return TableMetaTSDBBuilder.build(destination, "classpath:spring/tsdb/mysql-tsdb.xml");
                    } finally {
                        System.setProperty("canal.instance.tsdb.url", "");
                        System.setProperty("canal.instance.tsdb.dbUsername", "");
                        System.setProperty("canal.instance.tsdb.dbPassword", "");
                    }
                }
            });
            mysqlEventParser.setEnableTsdb(tsdbEnable);
        }
        eventParser = mysqlEventParser;
    } else if (type.isLocalBinlog()) {
        LocalBinlogEventParser localBinlogEventParser = new LocalBinlogEventParser();
        localBinlogEventParser.setDestination(destination);
        localBinlogEventParser.setBufferSize(parameters.getReceiveBufferSize());
        localBinlogEventParser.setConnectionCharset(parameters.getConnectionCharset());
        localBinlogEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
        localBinlogEventParser.setDirectory(parameters.getLocalBinlogDirectory());
        localBinlogEventParser.setProfilingEnabled(false);
        localBinlogEventParser.setDetectingEnable(parameters.getDetectingEnable());
        localBinlogEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
        localBinlogEventParser.setFilterTableError(parameters.getFilterTableError());
        localBinlogEventParser.setParallel(parameters.getParallel());
        // 数据库信息，反查表结构时需要
        if (!CollectionUtils.isEmpty(dbAddresses)) {
            localBinlogEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName()));
        }
        eventParser = localBinlogEventParser;
    } else if (type.isOracle()) {
        throw new CanalException("unsupport SourcingType for " + type);
    } else {
        throw new CanalException("unsupport SourcingType for " + type);
    }
    // add transaction support at 2012-12-06
    if (eventParser instanceof AbstractEventParser) {
        AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
        abstractEventParser.setTransactionSize(parameters.getTransactionSize());
        abstractEventParser.setLogPositionManager(initLogPositionManager());
        abstractEventParser.setAlarmHandler(getAlarmHandler());
        abstractEventParser.setEventSink(getEventSink());
        if (StringUtils.isNotEmpty(filter)) {
            AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(filter);
            abstractEventParser.setEventFilter(aviaterFilter);
        }
        // 设置黑名单
        if (StringUtils.isNotEmpty(parameters.getBlackFilter())) {
            AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(parameters.getBlackFilter());
            abstractEventParser.setEventBlackFilter(aviaterFilter);
        }
    }
    if (eventParser instanceof MysqlEventParser) {
        MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
        // 初始化haController，绑定与eventParser的关系，haController会控制eventParser
        CanalHAController haController = initHaController();
        mysqlEventParser.setHaController(haController);
    }
    return eventParser;
}

Example 4 with AuthenticationInfo

use of com.alibaba.otter.canal.parse.support.AuthenticationInfo in project canal by alibaba.

the class MysqlEventParser method doSwitch.

// 处理主备切换的逻辑
public void doSwitch() {
    AuthenticationInfo newRunningInfo = (runningInfo.equals(masterInfo) ? standbyInfo : masterInfo);
    this.doSwitch(newRunningInfo);
}

Example 5 with AuthenticationInfo

use of com.alibaba.otter.canal.parse.support.AuthenticationInfo in project canal by alibaba.

the class MysqlBinlogDumpPerformanceTest method main.

public static void main(String[] args) {
    final MysqlEventParser controller = new MysqlEventParser();
    final EntryPosition startPosition = new EntryPosition("mysql-bin.000007", 89796293L, 100L);
    controller.setConnectionCharset("UTF-8");
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setFilterQueryDml(true);
    controller.setMasterInfo(new AuthenticationInfo(new InetSocketAddress("100.81.154.142", 3306), "canal", "canal"));
    controller.setMasterPosition(startPosition);
    controller.setEnableTsdb(false);
    controller.setDestination("example");
    controller.setTsdbSpringXml("classpath:spring/tsdb/h2-tsdb.xml");
    // controller.setEventFilter(new AviaterRegexFilter("test\\..*"));
    // controller.setEventBlackFilter(new
    // AviaterRegexFilter("canal_tsdb\\..*"));
    controller.setParallel(true);
    controller.setParallelBufferSize(256);
    controller.setParallelThreadSize(16);
    controller.setIsGTIDMode(false);
    final AtomicLong sum = new AtomicLong(0);
    final AtomicLong last = new AtomicLong(0);
    final AtomicLong start = new AtomicLong(System.currentTimeMillis());
    final AtomicLong end = new AtomicLong(0);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {

        public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException, InterruptedException {
            sum.addAndGet(entrys.size());
            long current = sum.get();
            if (current - last.get() >= 100000) {
                end.set(System.currentTimeMillis());
                long tps = ((current - last.get()) * 1000) / (end.get() - start.get());
                System.out.println(" total : " + sum + " , cost : " + (end.get() - start.get()) + " , tps : " + tps);
                last.set(current);
                start.set(end.get());
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        @Override
        public LogPosition getLatestIndexBy(String destination) {
            return null;
        }

        @Override
        public void persistLogPosition(String destination, LogPosition logPosition) throws CanalParseException {
        }
    });
    controller.start();
    try {
        Thread.sleep(100 * 1000 * 1000L);
    } catch (InterruptedException e) {
    }
    controller.stop();
}