Examples with AviaterRegexFilter com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter used on opensource projects

Example 1 with AviaterRegexFilter

use of com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter in project canal by alibaba.

the class RdsBinlogEventParserProxyTest method test_timestamp.

@Test
public void test_timestamp() throws InterruptedException {
    final TimeoutChecker timeoutChecker = new TimeoutChecker(3000 * 1000);
    final AtomicLong entryCount = new AtomicLong(0);
    final EntryPosition entryPosition = new EntryPosition();
    final RdsBinlogEventParserProxy controller = new RdsBinlogEventParserProxy();
    Calendar calendar = Calendar.getInstance();
    calendar.add(Calendar.HOUR_OF_DAY, -24 * 4);
    final EntryPosition defaultPosition = buildPosition(null, null, calendar.getTimeInMillis());
    controller.setSlaveId(3344L);
    controller.setDetectingEnable(false);
    controller.setDetectingSQL(DETECTING_SQL);
    controller.setMasterInfo(buildAuthentication());
    controller.setMasterPosition(defaultPosition);
    controller.setInstanceId("");
    controller.setAccesskey("");
    controller.setSecretkey("");
    controller.setDirectory("/tmp/binlog");
    controller.setEventBlackFilter(new AviaterRegexFilter("mysql\\.*"));
    controller.setFilterTableError(true);
    controller.setBatchFileSize(4);
    controller.setEventSink(new AbstractCanalEventSinkTest<List<CanalEntry.Entry>>() {

        @Override
        public boolean sink(List<CanalEntry.Entry> entrys, InetSocketAddress remoteAddress, String destination) throws CanalSinkException {
            for (CanalEntry.Entry entry : entrys) {
                if (entry.getEntryType() != CanalEntry.EntryType.HEARTBEAT) {
                    entryCount.incrementAndGet();
                    String logfilename = entry.getHeader().getLogfileName();
                    long logfileoffset = entry.getHeader().getLogfileOffset();
                    long executeTime = entry.getHeader().getExecuteTime();
                    entryPosition.setJournalName(logfilename);
                    entryPosition.setPosition(logfileoffset);
                    entryPosition.setTimestamp(executeTime);
                    break;
                }
            }
            return true;
        }
    });
    controller.setLogPositionManager(new AbstractLogPositionManager() {

        private LogPosition logPosition;

        public void persistLogPosition(String destination, LogPosition logPosition) {
            this.logPosition = logPosition;
        }

        public LogPosition getLatestIndexBy(String destination) {
            return logPosition;
        }
    });
    controller.start();
    timeoutChecker.waitForIdle();
    if (controller.isStart()) {
        controller.stop();
    }
    // check
    Assert.assertTrue(entryCount.get() > 0);
    // 对比第一条数据和起始的position相同
    Assert.assertEquals(entryPosition.getJournalName(), "mysql-bin.000001");
    Assert.assertTrue(entryPosition.getPosition() <= 6163L);
    Assert.assertTrue(entryPosition.getTimestamp() <= defaultPosition.getTimestamp());
}

Example 2 with AviaterRegexFilter

use of com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter in project canal by alibaba.

the class CanalInstanceWithManager method doInitEventParser.

private CanalEventParser doInitEventParser(SourcingType type, List<InetSocketAddress> dbAddresses) {
    CanalEventParser eventParser;
    if (type.isMysql()) {
        MysqlEventParser mysqlEventParser = null;
        if (StringUtils.isNotEmpty(parameters.getRdsAccesskey()) && StringUtils.isNotEmpty(parameters.getRdsSecretkey()) && StringUtils.isNotEmpty(parameters.getRdsInstanceId())) {
            mysqlEventParser = new RdsBinlogEventParserProxy();
            ((RdsBinlogEventParserProxy) mysqlEventParser).setAccesskey(parameters.getRdsAccesskey());
            ((RdsBinlogEventParserProxy) mysqlEventParser).setSecretkey(parameters.getRdsSecretkey());
            ((RdsBinlogEventParserProxy) mysqlEventParser).setInstanceId(parameters.getRdsInstanceId());
        } else {
            mysqlEventParser = new MysqlEventParser();
        }
        mysqlEventParser.setDestination(destination);
        // 编码参数
        mysqlEventParser.setConnectionCharset(parameters.getConnectionCharset());
        mysqlEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
        // 网络相关参数
        mysqlEventParser.setDefaultConnectionTimeoutInSeconds(parameters.getDefaultConnectionTimeoutInSeconds());
        mysqlEventParser.setSendBufferSize(parameters.getSendBufferSize());
        mysqlEventParser.setReceiveBufferSize(parameters.getReceiveBufferSize());
        // 心跳检查参数
        mysqlEventParser.setDetectingEnable(parameters.getDetectingEnable());
        mysqlEventParser.setDetectingSQL(parameters.getDetectingSQL());
        mysqlEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
        // 数据库信息参数
        mysqlEventParser.setSlaveId(parameters.getSlaveId());
        if (!CollectionUtils.isEmpty(dbAddresses)) {
            mysqlEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName()));
            if (dbAddresses.size() > 1) {
                mysqlEventParser.setStandbyInfo(new AuthenticationInfo(dbAddresses.get(1), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName()));
            }
        }
        if (!CollectionUtils.isEmpty(parameters.getPositions())) {
            EntryPosition masterPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(0), EntryPosition.class);
            // binlog位置参数
            mysqlEventParser.setMasterPosition(masterPosition);
            if (parameters.getPositions().size() > 1) {
                EntryPosition standbyPosition = JsonUtils.unmarshalFromString(parameters.getPositions().get(1), EntryPosition.class);
                mysqlEventParser.setStandbyPosition(standbyPosition);
            }
        }
        mysqlEventParser.setFallbackIntervalInSeconds(parameters.getFallbackIntervalInSeconds());
        mysqlEventParser.setProfilingEnabled(false);
        mysqlEventParser.setFilterTableError(parameters.getFilterTableError());
        mysqlEventParser.setParallel(parameters.getParallel());
        mysqlEventParser.setIsGTIDMode(BooleanUtils.toBoolean(parameters.getGtidEnable()));
        // tsdb
        if (parameters.getTsdbSnapshotInterval() != null) {
            mysqlEventParser.setTsdbSnapshotInterval(parameters.getTsdbSnapshotInterval());
        }
        if (parameters.getTsdbSnapshotExpire() != null) {
            mysqlEventParser.setTsdbSnapshotExpire(parameters.getTsdbSnapshotExpire());
        }
        boolean tsdbEnable = BooleanUtils.toBoolean(parameters.getTsdbEnable());
        if (tsdbEnable) {
            mysqlEventParser.setTableMetaTSDBFactory(new DefaultTableMetaTSDBFactory() {

                @Override
                public void destory(String destination) {
                    TableMetaTSDBBuilder.destory(destination);
                }

                @Override
                public TableMetaTSDB build(String destination, String springXml) {
                    try {
                        System.setProperty("canal.instance.tsdb.url", parameters.getTsdbJdbcUrl());
                        System.setProperty("canal.instance.tsdb.dbUsername", parameters.getTsdbJdbcUserName());
                        System.setProperty("canal.instance.tsdb.dbPassword", parameters.getTsdbJdbcPassword());
                        return TableMetaTSDBBuilder.build(destination, "classpath:spring/tsdb/mysql-tsdb.xml");
                    } finally {
                        System.setProperty("canal.instance.tsdb.url", "");
                        System.setProperty("canal.instance.tsdb.dbUsername", "");
                        System.setProperty("canal.instance.tsdb.dbPassword", "");
                    }
                }
            });
            mysqlEventParser.setEnableTsdb(tsdbEnable);
        }
        eventParser = mysqlEventParser;
    } else if (type.isLocalBinlog()) {
        LocalBinlogEventParser localBinlogEventParser = new LocalBinlogEventParser();
        localBinlogEventParser.setDestination(destination);
        localBinlogEventParser.setBufferSize(parameters.getReceiveBufferSize());
        localBinlogEventParser.setConnectionCharset(parameters.getConnectionCharset());
        localBinlogEventParser.setConnectionCharsetNumber(parameters.getConnectionCharsetNumber());
        localBinlogEventParser.setDirectory(parameters.getLocalBinlogDirectory());
        localBinlogEventParser.setProfilingEnabled(false);
        localBinlogEventParser.setDetectingEnable(parameters.getDetectingEnable());
        localBinlogEventParser.setDetectingIntervalInSeconds(parameters.getDetectingIntervalInSeconds());
        localBinlogEventParser.setFilterTableError(parameters.getFilterTableError());
        localBinlogEventParser.setParallel(parameters.getParallel());
        // 数据库信息，反查表结构时需要
        if (!CollectionUtils.isEmpty(dbAddresses)) {
            localBinlogEventParser.setMasterInfo(new AuthenticationInfo(dbAddresses.get(0), parameters.getDbUsername(), parameters.getDbPassword(), parameters.getDefaultDatabaseName()));
        }
        eventParser = localBinlogEventParser;
    } else if (type.isOracle()) {
        throw new CanalException("unsupport SourcingType for " + type);
    } else {
        throw new CanalException("unsupport SourcingType for " + type);
    }
    // add transaction support at 2012-12-06
    if (eventParser instanceof AbstractEventParser) {
        AbstractEventParser abstractEventParser = (AbstractEventParser) eventParser;
        abstractEventParser.setTransactionSize(parameters.getTransactionSize());
        abstractEventParser.setLogPositionManager(initLogPositionManager());
        abstractEventParser.setAlarmHandler(getAlarmHandler());
        abstractEventParser.setEventSink(getEventSink());
        if (StringUtils.isNotEmpty(filter)) {
            AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(filter);
            abstractEventParser.setEventFilter(aviaterFilter);
        }
        // 设置黑名单
        if (StringUtils.isNotEmpty(parameters.getBlackFilter())) {
            AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(parameters.getBlackFilter());
            abstractEventParser.setEventBlackFilter(aviaterFilter);
        }
    }
    if (eventParser instanceof MysqlEventParser) {
        MysqlEventParser mysqlEventParser = (MysqlEventParser) eventParser;
        // 初始化haController，绑定与eventParser的关系，haController会控制eventParser
        CanalHAController haController = initHaController();
        mysqlEventParser.setHaController(haController);
    }
    return eventParser;
}

Example 3 with AviaterRegexFilter

use of com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter in project canal by alibaba.

the class AbstractMysqlEventParser method buildParser.

protected BinlogParser buildParser() {
    LogEventConvert convert = new LogEventConvert();
    if (eventFilter != null && eventFilter instanceof AviaterRegexFilter) {
        convert.setNameFilter((AviaterRegexFilter) eventFilter);
    }
    if (eventBlackFilter != null && eventBlackFilter instanceof AviaterRegexFilter) {
        convert.setNameBlackFilter((AviaterRegexFilter) eventBlackFilter);
    }
    convert.setFieldFilterMap(getFieldFilterMap());
    convert.setFieldBlackFilterMap(getFieldBlackFilterMap());
    convert.setCharset(connectionCharset);
    convert.setFilterQueryDcl(filterQueryDcl);
    convert.setFilterQueryDml(filterQueryDml);
    convert.setFilterQueryDdl(filterQueryDdl);
    convert.setFilterRows(filterRows);
    convert.setFilterTableError(filterTableError);
    convert.setUseDruidDdlFilter(useDruidDdlFilter);
    return convert;
}

Example 4 with AviaterRegexFilter

use of com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter in project canal by alibaba.

the class AviaterFilterTest method test_regex.

@Test
public void test_regex() {
    AviaterRegexFilter filter = new AviaterRegexFilter("s1\\..*,s2\\..*");
    boolean result = filter.filter("s1.t1");
    Assert.assertEquals(true, result);
    result = filter.filter("s1.t2");
    Assert.assertEquals(true, result);
    result = filter.filter("");
    Assert.assertEquals(true, result);
    result = filter.filter("s12.t1");
    Assert.assertEquals(false, result);
    result = filter.filter("s2.t2");
    Assert.assertEquals(true, result);
    result = filter.filter("s3.t2");
    Assert.assertEquals(false, result);
    result = filter.filter("S1.S2");
    Assert.assertEquals(true, result);
    result = filter.filter("S2.S1");
    Assert.assertEquals(true, result);
    AviaterRegexFilter filter2 = new AviaterRegexFilter("s1\\..*,s2.t1");
    result = filter2.filter("s1.t1");
    Assert.assertEquals(true, result);
    result = filter2.filter("s1.t2");
    Assert.assertEquals(true, result);
    result = filter2.filter("s2.t1");
    Assert.assertEquals(true, result);
    AviaterRegexFilter filter3 = new AviaterRegexFilter("foooo,f.*t");
    result = filter3.filter("fooooot");
    Assert.assertEquals(true, result);
    AviaterRegexFilter filter4 = new AviaterRegexFilter("otter2.otter_stability1|otter1.otter_stability1|retl.retl_mark|retl.retl_buffer|retl.xdual");
    result = filter4.filter("otter1.otter_stability1");
    Assert.assertEquals(true, result);
}

Example 5 with AviaterRegexFilter

use of com.alibaba.otter.canal.filter.aviater.AviaterRegexFilter in project canal by alibaba.

the class AbstractCanalInstance method subscribeChange.

@Override
public boolean subscribeChange(ClientIdentity identity) {
    if (StringUtils.isNotEmpty(identity.getFilter())) {
        logger.info("subscribe filter change to " + identity.getFilter());
        AviaterRegexFilter aviaterFilter = new AviaterRegexFilter(identity.getFilter());
        boolean isGroup = (eventParser instanceof GroupEventParser);
        if (isGroup) {
            // 处理group的模式
            List<CanalEventParser> eventParsers = ((GroupEventParser) eventParser).getEventParsers();
            for (CanalEventParser singleEventParser : eventParsers) {
                // 需要遍历启动
                if (singleEventParser instanceof AbstractEventParser) {
                    ((AbstractEventParser) singleEventParser).setEventFilter(aviaterFilter);
                }
            }
        } else {
            if (eventParser instanceof AbstractEventParser) {
                ((AbstractEventParser) eventParser).setEventFilter(aviaterFilter);
            }
        }
    }
    // 后续内存版的一对多分发，可以考虑
    return true;
}