Examples with CanalEventParser com.alibaba.otter.canal.parse.CanalEventParser used on opensource projects

Example 1 with CanalEventParser

use of com.alibaba.otter.canal.parse.CanalEventParser in project canal by alibaba.

the class BaseCanalServerWithEmbededTest method testSwitch.

// @Test
public void testSwitch() {
    int maxEmptyCount = 10;
    int emptyCount = 0;
    int totalCount = 0;
    int thresold = 50;
    int batchSize = 11;
    server.subscribe(clientIdentity);
    while (emptyCount < maxEmptyCount) {
        Message message = server.get(clientIdentity, batchSize);
        if (CollectionUtils.isEmpty(message.getEntries())) {
            emptyCount++;
            try {
                Thread.sleep(emptyCount * 300L);
            } catch (InterruptedException e) {
                Assert.fail();
            }
            System.out.println("empty count : " + emptyCount);
        } else {
            emptyCount = 0;
            totalCount += message.getEntries().size();
            if ((totalCount + 1) % 100 >= thresold && (totalCount + 1) % 100 <= thresold + batchSize) {
                CanalEventParser eventParser = server.getCanalInstances().get(DESTINATION).getEventParser();
                if (eventParser instanceof CanalHASwitchable) {
                    // 执行切换
                    ((CanalHASwitchable) eventParser).doSwitch();
                    try {
                        // 等待parser启动
                        Thread.sleep(5 * 1000);
                    } catch (InterruptedException e) {
                        Assert.fail();
                    }
                }
            }
        }
    }
    System.out.println("!!!!!! testGet totalCount : " + totalCount);
    server.unsubscribe(clientIdentity);
}

Example 2 with CanalEventParser

use of com.alibaba.otter.canal.parse.CanalEventParser in project otter by alibaba.

the class CanalEmbedSelector method start.

public void start() {
    if (running) {
        return;
    }
    // 获取destination/filter参数
    Pipeline pipeline = configClientService.findPipeline(pipelineId);
    filter = CanalFilterSupport.makeFilterExpression(pipeline);
    destination = pipeline.getParameters().getDestinationName();
    batchSize = pipeline.getParameters().getMainstemBatchsize();
    batchTimeout = pipeline.getParameters().getBatchTimeout();
    ddlSync = pipeline.getParameters().getDdlSync();
    final boolean syncFull = pipeline.getParameters().getSyncMode().isRow() || pipeline.getParameters().isEnableRemedy();
    // 暂时使用skip load代替
    filterTableError = pipeline.getParameters().getSkipSelectException();
    if (pipeline.getParameters().getDumpSelector() != null) {
        dump = pipeline.getParameters().getDumpSelector();
    }
    if (pipeline.getParameters().getDumpSelectorDetail() != null) {
        dumpDetail = pipeline.getParameters().getDumpSelectorDetail();
    }
    canalServer.setCanalInstanceGenerator(new CanalInstanceGenerator() {

        public CanalInstance generate(String destination) {
            Canal canal = canalConfigClient.findCanal(destination);
            final OtterAlarmHandler otterAlarmHandler = new OtterAlarmHandler();
            otterAlarmHandler.setPipelineId(pipelineId);
            // 注入一下spring资源
            OtterContextLocator.autowire(otterAlarmHandler);
            // 设置下slaveId，保证多个piplineId下重复引用时不重复
            // 默认基数
            long slaveId = 10000;
            if (canal.getCanalParameter().getSlaveId() != null) {
                slaveId = canal.getCanalParameter().getSlaveId();
            }
            canal.getCanalParameter().setSlaveId(slaveId + pipelineId);
            canal.getCanalParameter().setDdlIsolation(ddlSync);
            canal.getCanalParameter().setFilterTableError(filterTableError);
            canal.getCanalParameter().setMemoryStorageRawEntry(false);
            CanalInstanceWithManager instance = new CanalInstanceWithManager(canal, filter) {

                protected CanalHAController initHaController() {
                    HAMode haMode = parameters.getHaMode();
                    if (haMode.isMedia()) {
                        return new MediaHAController(parameters.getMediaGroup(), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName());
                    } else {
                        return super.initHaController();
                    }
                }

                protected void startEventParserInternal(CanalEventParser parser, boolean isGroup) {
                    super.startEventParserInternal(parser, isGroup);
                    if (eventParser instanceof MysqlEventParser) {
                        // 设置支持的类型
                        ((MysqlEventParser) eventParser).setSupportBinlogFormats("ROW");
                        if (syncFull) {
                            ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL");
                        } else {
                            ((MysqlEventParser) eventParser).setSupportBinlogImages("FULL,MINIMAL");
                        }
                        MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
                        // otter先使用简单的模式
                        mysqlEventParser.setParallel(false);
                        CanalHAController haController = mysqlEventParser.getHaController();
                        if (haController instanceof MediaHAController) {
                            if (isGroup) {
                                throw new CanalException("not support group database use media HA");
                            }
                            ((MediaHAController) haController).setCanalHASwitchable(mysqlEventParser);
                        }
                        if (!haController.isStart()) {
                            haController.start();
                        }
                        // 基于media的Ha，直接从tddl中获取数据库信息
                        if (haController instanceof MediaHAController) {
                            AuthenticationInfo authenticationInfo = ((MediaHAController) haController).getAvailableAuthenticationInfo();
                            ((MysqlEventParser) eventParser).setMasterInfo(authenticationInfo);
                        }
                    }
                }
            };
            instance.setAlarmHandler(otterAlarmHandler);
            CanalEventSink eventSink = instance.getEventSink();
            if (eventSink instanceof AbstractCanalEventSink) {
                handler = new OtterDownStreamHandler();
                handler.setPipelineId(pipelineId);
                handler.setDetectingIntervalInSeconds(canal.getCanalParameter().getDetectingIntervalInSeconds());
                // 注入一下spring资源
                OtterContextLocator.autowire(handler);
                // 添加到开头
                ((AbstractCanalEventSink) eventSink).addHandler(handler, 0);
                handler.start();
            }
            return instance;
        }
    });
    canalServer.start();
    canalServer.start(destination);
    this.clientIdentity = new ClientIdentity(destination, pipeline.getParameters().getMainstemClientId(), filter);
    // 发起一次订阅
    canalServer.subscribe(clientIdentity);
    running = true;
}

Example 3 with CanalEventParser

use of com.alibaba.otter.canal.parse.CanalEventParser in project canal by alibaba.

the class ParserCollector method register.

@Override
public void register(CanalInstance instance) {
    final String destination = instance.getDestination();
    ParserMetricsHolder holder;
    CanalEventParser parser = instance.getEventParser();
    if (parser instanceof AbstractMysqlEventParser) {
        holder = singleHolder(destination, (AbstractMysqlEventParser) parser, "0");
    } else if (parser instanceof GroupEventParser) {
        holder = groupHolder(destination, (GroupEventParser) parser);
    } else {
        throw new IllegalArgumentException("CanalEventParser must be either AbstractMysqlEventParser or GroupEventParser.");
    }
    Preconditions.checkNotNull(holder);
    ParserMetricsHolder old = instances.put(destination, holder);
    if (old != null) {
        logger.warn("Remove stale ParserCollector for instance {}.", destination);
    }
}

Example 4 with CanalEventParser

use of com.alibaba.otter.canal.parse.CanalEventParser in project canal by alibaba.

the class CanalInstanceWithManager method doInitEventParser.

private CanalEventParser doInitEventParser(SourcingType type, List<InetSocketAddress> dbAddresses) {
    CanalEventParser eventParser;
    if (type.isMysql()) {
        MysqlEventParser mysqlEventParser = null;
        if (StringUtils.isNotEmpty(parameters.getRdsAccesskey()) && StringUtils.isNotEmpty(parameters.getRdsSecretkey()) && StringUtils.isNotEmpty(parameters.getRdsInstanceId())) {
            mysqlEventParser = new RdsBinlogEventParserProxy();
            ((RdsBinlogEventParserProxy) mysqlEventParser).setAccesskey(parameters.getRdsAccesskey());
            ((RdsBinlogEventParserProxy) mysqlEventParser).setSecretkey(parameters.getRdsSecretkey());
            ((RdsBinlogEventParserProxy) mysqlEventParser).setInstanceId(parameters.getRdsInstanceId());
        } else {
            mysqlEventParser = new MysqlEventParser();
        }
        mysqlEventParser.setDestination(destination);
        // 编码参数
        mysqlEventParser.setConnectionCharset(parameters.getConnectionCharset());
        mysqlEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
        // 网络相关参数
        mysqlEventParser.setDefaultConnectionTimeoutInSeconds(parameters.getDefaultConnectionTimeoutInSeconds());
        mysqlEventParser.setSendBufferSize(parameters.getSendBufferSize());
        mysqlEventParser.setReceiveBufferSize(parameters.getReceiveBufferSize());
        // 心跳检查参数
        mysqlEventParser.setDetectingEnable(parameters.getDetectingEnable());
        mysqlEventParser.setDetectingSQL(parameters.getDetectingSQL());
        mysqlEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
        // 数据库信息参数
        mysqlEventParser.setSlaveId(parameters.getSlaveId());
        if (!CollectionUtils.isEmpty(dbAddresses)) {
            mysqlEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName()));
            if (dbAddresses.size() > 1) {
                mysqlEventParser.setStandbyInfo(new AuthenticationInfo(dbAddresses.get(1), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName()));
            }
        }
        if (!CollectionUtils.isEmpty(parameters.getPositions())) {
            EntryPosition masterPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(0), EntryPosition.class);
            // binlog位置参数
            mysqlEventParser.setMasterPosition(masterPosition);
            if (parameters.getPositions().size() > 1) {
                EntryPosition standbyPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(1), EntryPosition.class);
                mysqlEventParser.setStandbyPosition(standbyPosition);
            }
        }
        mysqlEventParser.setFallbackIntervalInSeconds(parameters.getFallbackIntervalInSeconds());
        mysqlEventParser.setProfilingEnabled(false);
        mysqlEventParser.setFilterTableError(parameters.getFilterTableError());
        mysqlEventParser.setParallel(parameters.getParallel());
        mysqlEventParser.setIsGTIDMode(BooleanUtils.toBoolean(parameters.getGtidEnable()));
        // tsdb
        if (parameters.getTsdbSnapshotInterval() != null) {
            mysqlEventParser.setTsdbSnapshotInterval(parameters.getTsdbSnapshotInterval());
        }
        if (parameters.getTsdbSnapshotExpire() != null) {
            mysqlEventParser.setTsdbSnapshotExpire(parameters.getTsdbSnapshotExpire());
        }
        boolean tsdbEnable = BooleanUtils.toBoolean(parameters.getTsdbEnable());
        if (tsdbEnable) {
            mysqlEventParser.setTableMetaTSDBFactory(new DefaultTableMetaTSDBFactory() {

                @Override
                public void destory(String destination) {
                    TableMetaTSDBBuilder.destory(destination);
                }

                @Override
                public TableMetaTSDB build(String destination, String springXml) {
                    try {
                        System.setProperty("canal.instance.tsdb.url", parameters.getTsdbJdbcUrl());
                        System.setProperty("canal.instance.tsdb.dbUsername", parameters.getTsdbJdbcUserName());
                        System.setProperty("canal.instance.tsdb.dbPassword", parameters.getTsdbJdbcPassword());
                        return TableMetaTSDBBuilder.build(destination, "classpath:spring/tsdb/mysql-tsdb.xml");
                    } finally {
                        System.setProperty("canal.instance.tsdb.url", "");
                        System.setProperty("canal.instance.tsdb.dbUsername", "");
                        System.setProperty("canal.instance.tsdb.dbPassword", "");
                    }
                }
            });
            mysqlEventParser.setEnableTsdb(tsdbEnable);
        }
        eventParser = mysqlEventParser;
    } else if (type.isLocalBinlog()) {
        LocalBinlogEventParser localBinlogEventParser = new LocalBinlogEventParser();
        localBinlogEventParser.setDestination(destination);
        localBinlogEventParser.setBufferSize(parameters.getReceiveBufferSize());
        localBinlogEventParser.setConnectionCharset(parameters.getConnectionCharset());
        localBinlogEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
        localBinlogEventParser.setDirectory(parameters.getLocalBinlogDirectory());
        localBinlogEventParser.setProfilingEnabled(false);
        localBinlogEventParser.setDetectingEnable(parameters.getDetectingEnable());
        localBinlogEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
        localBinlogEventParser.setFilterTableError(parameters.getFilterTableError());
        localBinlogEventParser.setParallel(parameters.getParallel());
        // 数据库信息，反查表结构时需要
        if (!CollectionUtils.isEmpty(dbAddresses)) {
            localBinlogEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName()));
        }
        eventParser = localBinlogEventParser;
    } else if (type.isOracle()) {
        throw new CanalException("unsupport SourcingType for " + type);
    } else {
        throw new CanalException("unsupport SourcingType for " + type);
    }
    // add transaction support at 2012-12-06
    if (eventParser instanceof AbstractEventParser) {
        AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
        abstractEventParser.setTransactionSize(parameters.getTransactionSize());
        abstractEventParser.setLogPositionManager(initLogPositionManager());
        abstractEventParser.setAlarmHandler(getAlarmHandler());
        abstractEventParser.setEventSink(getEventSink());
        if (StringUtils.isNotEmpty(filter)) {
            AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(filter);
            abstractEventParser.setEventFilter(aviaterFilter);
        }
        // 设置黑名单
        if (StringUtils.isNotEmpty(parameters.getBlackFilter())) {
            AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(parameters.getBlackFilter());
            abstractEventParser.setEventBlackFilter(aviaterFilter);
        }
    }
    if (eventParser instanceof MysqlEventParser) {
        MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
        // 初始化haController，绑定与eventParser的关系，haController会控制eventParser
        CanalHAController haController = initHaController();
        mysqlEventParser.setHaController(haController);
    }
    return eventParser;
}

Example 5 with CanalEventParser

use of com.alibaba.otter.canal.parse.CanalEventParser in project canal by alibaba.

the class CanalInstanceWithManager method initEventParser.

protected void initEventParser() {
    logger.info("init eventParser begin...");
    SourcingType type = parameters.getSourcingType();
    List<List<DataSourcing>> groupDbAddresses = parameters.getGroupDbAddresses();
    if (!CollectionUtils.isEmpty(groupDbAddresses)) {
        // 取第一个分组的数量，主备分组的数量必须一致
        int size = groupDbAddresses.get(0).size();
        List<CanalEventParser> eventParsers = new ArrayList<>();
        for (int i = 0; i < size; i++) {
            List<InetSocketAddress> dbAddress = new ArrayList<>();
            SourcingType lastType = null;
            for (List<DataSourcing> groupDbAddress : groupDbAddresses) {
                if (lastType != null && !lastType.equals(groupDbAddress.get(i).getType())) {
                    throw new CanalException(String.format("master/slave Sourcing type is unmatch. %s vs %s", lastType, groupDbAddress.get(i).getType()));
                }
                lastType = groupDbAddress.get(i).getType();
                dbAddress.add(groupDbAddress.get(i).getDbAddress());
            }
            // 初始化其中的一个分组parser
            eventParsers.add(doInitEventParser(lastType, dbAddress));
        }
        if (eventParsers.size() > 1) {
            // 如果存在分组，构造分组的parser
            GroupEventParser groupEventParser = new GroupEventParser();
            groupEventParser.setEventParsers(eventParsers);
            this.eventParser = groupEventParser;
        } else {
            this.eventParser = eventParsers.get(0);
        }
    } else {
        // 创建一个空数据库地址的parser，可能使用了tddl指定地址，启动的时候才会从tddl获取地址
        this.eventParser = doInitEventParser(type, new ArrayList<>());
    }
    logger.info("init eventParser end! \n\t load CanalEventParser:{}", eventParser.getClass().getName());
}