Examples with ReaderSource com.qlangtech.tis.realtime.ReaderSource used on opensource projects

Example 1 with ReaderSource

use of com.qlangtech.tis.realtime.ReaderSource in project plugins by qlangtech.

the class FlinkCDCMongoDBSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName dataxName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        DataXMongodbReader mongoReader = (DataXMongodbReader) dataSource;
        MangoDBDataSourceFactory dsFactory = mongoReader.getDsFactory();
        List<ReaderSource> sourceFunctions = Lists.newArrayList();
        MongoDBSource.Builder<DTO> builder = MongoDBSource.<DTO>builder().hosts(dsFactory.address).database(dsFactory.dbName).collection(mongoReader.collectionName).connectionOptions(sourceFactory.connectionOptions).errorsTolerance(sourceFactory.errorsTolerance).username(dsFactory.getUserName()).password(dsFactory.getPassword()).deserializer(new TISDeserializationSchema());
        if (sourceFactory.errorsLogEnable != null) {
            builder.errorsLogEnable(sourceFactory.errorsLogEnable);
        }
        if (sourceFactory.copyExisting != null) {
            builder.copyExisting(sourceFactory.copyExisting);
        }
        if (sourceFactory.copyExistingMaxThreads != null) {
            builder.copyExistingMaxThreads(sourceFactory.copyExistingMaxThreads);
        }
        if (sourceFactory.copyExistingQueueSize != null) {
            builder.copyExistingMaxThreads(sourceFactory.copyExistingQueueSize);
        }
        if (sourceFactory.pollMaxBatchSize != null) {
            builder.copyExistingMaxThreads(sourceFactory.pollMaxBatchSize);
        }
        if (sourceFactory.pollAwaitTimeMillis != null) {
            builder.copyExistingMaxThreads(sourceFactory.pollAwaitTimeMillis);
        }
        if (sourceFactory.heartbeatIntervalMillis != null) {
            builder.copyExistingMaxThreads(sourceFactory.heartbeatIntervalMillis);
        }
        SourceFunction<DTO> source = builder.build();
        // MongoDBSource.<DTO>builder()
        // .hosts(dsFactory.address)
        // .database(dsFactory.dbName)
        // .collection(mongoReader.collectionName)
        // .connectionOptions(sourceFactory.connectionOptions)
        // .errorsTolerance(sourceFactory.errorsTolerance)
        // .errorsLogEnable(sourceFactory.errorsLogEnable)
        // .copyExisting(sourceFactory.copyExisting)
        // .copyExistingPipeline(sourceFactory.copyExistingPipeline)
        // .copyExistingMaxThreads(sourceFactory.copyExistingMaxThreads)
        // .copyExistingQueueSize(sourceFactory.copyExistingQueueSize)
        // .pollMaxBatchSize(sourceFactory.pollMaxBatchSize)
        // .pollAwaitTimeMillis(sourceFactory.pollAwaitTimeMillis)
        // .heartbeatIntervalMillis(sourceFactory.heartbeatIntervalMillis)
        // //.port(dsFactory.port)
        // // .databaseList(dbs.toArray(new String[dbs.size()])) // monitor all tables under inventory database
        // //                              .tableList(tbs.toArray(new String[tbs.size()]))
        // .username(dsFactory.getUserName())
        // .password(dsFactory.getPassword())
        // //                              .startupOptions(sourceFactory.getStartupOptions())
        // //.debeziumProperties(debeziumProperties)
        // .deserializer(new TISDeserializationSchema()) // converts SourceRecord to JSON String
        // .build();
        sourceFunctions.add(new ReaderSource(dsFactory.address + "_" + dsFactory.dbName + "_" + mongoReader.collectionName, source));
        SourceChannel sourceChannel = new SourceChannel(sourceFunctions);
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(dataxName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}

Example 2 with ReaderSource

use of com.qlangtech.tis.realtime.ReaderSource in project plugins by qlangtech.

the class FlinkCDCMysqlSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName dataxName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        // TabColIndexer colIndexer = new TabColIndexer(tabs);
        // TISDeserializationSchema deserializationSchema
        // = new TISDeserializationSchema(new MySQLSourceValConvert(colIndexer));
        TISDeserializationSchema deserializationSchema = new TISDeserializationSchema();
        BasicDataXRdbmsReader rdbmsReader = (BasicDataXRdbmsReader) dataSource;
        BasicDataSourceFactory dsFactory = (BasicDataSourceFactory) rdbmsReader.getDataSourceFactory();
        SourceChannel sourceChannel = new SourceChannel(SourceChannel.getSourceFunction(dsFactory, tabs, (dbHost, dbs, tbs, debeziumProperties) -> {
            DateTimeConverter.setDatetimeConverters(MySqlDateTimeConverter.class.getName(), debeziumProperties);
            String[] databases = dbs.toArray(new String[dbs.size()]);
            return Collections.singletonList(new ReaderSource(dbHost + ":" + dsFactory.port + ":" + dbs.stream().collect(Collectors.joining("_")), MySqlSource.<DTO>builder().hostname(dbHost).port(dsFactory.port).databaseList(// monitor all tables under inventory database
            databases).tableList(tbs.toArray(new String[tbs.size()])).serverTimeZone(BasicDataSourceFactory.DEFAULT_SERVER_TIME_ZONE.getId()).username(dsFactory.getUserName()).password(dsFactory.getPassword()).startupOptions(sourceFactory.getStartupOptions()).debeziumProperties(debeziumProperties).deserializer(// converts SourceRecord to JSON String
            deserializationSchema).build()));
        }));
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(dataxName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}

Example 3 with ReaderSource

use of com.qlangtech.tis.realtime.ReaderSource in project plugins by qlangtech.

the class SourceChannel method getSourceFunction.

// https://ververica.github.io/flink-cdc-connectors/master/
public static List<ReaderSource> getSourceFunction(BasicDataSourceFactory dsFactory, Function<DBTable, String> tabnameCreator, List<ISelectedTab> tabs, ReaderSourceCreator sourceFunctionCreator) {
    try {
        DBConfig dbConfig = dsFactory.getDbConfig();
        List<ReaderSource> sourceFuncs = Lists.newArrayList();
        Map<String, List<String>> ip2dbs = Maps.newHashMap();
        Map<String, List<ISelectedTab>> db2tabs = Maps.newHashMap();
        dbConfig.vistDbName((config, ip, dbName) -> {
            List<String> dbs = ip2dbs.get(ip);
            if (dbs == null) {
                dbs = Lists.newArrayList();
                ip2dbs.put(ip, dbs);
            }
            dbs.add(dbName);
            if (db2tabs.get(dbName) == null) {
                db2tabs.put(dbName, tabs);
            }
            return false;
        });
        for (Map.Entry<String, List<String>> /**
         *dbs
         */
        entry : ip2dbs.entrySet()) {
            // Set<String> tbs = entry.getValue().stream().flatMap(
            // (dbName) -> db2tabs.get(dbName).stream().map((tab) -> dbName + "." + tab.getName())).collect(Collectors.toSet());
            Set<String> tbs = entry.getValue().stream().flatMap((dbName) -> db2tabs.get(dbName).stream().map((tab) -> {
                // return (dsSchemaSupport ? ((BasicDataSourceFactory.ISchemaSupported) dsFactory).getDBSchema() : dbName) + "." + tab.getName();
                return tabnameCreator.apply(new DBTable(dbName, tab));
            })).collect(Collectors.toSet());
            Properties debeziumProperties = new Properties();
            // do not use lock
            debeziumProperties.put("snapshot.locking.mode", "none");
            String dbHost = entry.getKey();
            List<String> dbs = entry.getValue();
            sourceFuncs.addAll(sourceFunctionCreator.create(dbHost, dbs, tbs, debeziumProperties));
        }
        return sourceFuncs;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 4 with ReaderSource

use of com.qlangtech.tis.realtime.ReaderSource in project plugins by qlangtech.

the class FlinkCDCOracleSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName channalName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        BasicDataXRdbmsReader reader = (BasicDataXRdbmsReader) dataSource;
        BasicDataSourceFactory f = (BasicDataSourceFactory) reader.getDataSourceFactory();
        SourceChannel sourceChannel = new SourceChannel(SourceChannel.getSourceFunction(f, (tab) -> tab.getTabName(), tabs, (dbHost, dbs, tbs, debeziumProperties) -> {
            return dbs.stream().map((databaseName) -> {
                SourceFunction<DTO> sourceFunction = OracleSource.<DTO>builder().hostname(dbHost).debeziumProperties(debeziumProperties).port(f.port).startupOptions(sourceFactory.getStartupOptions()).database(// monitor XE database
                StringUtils.upperCase(f.dbName)).tableList(// monitor products table
                tbs.toArray(new String[tbs.size()])).username(f.getUserName()).password(f.getPassword()).deserializer(// converts SourceRecord to JSON String
                new TISDeserializationSchema()).build();
                return new ReaderSource(dbHost + ":" + f.port + "_" + databaseName, sourceFunction);
            }).collect(Collectors.toList());
        }));
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(channalName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}

Example 5 with ReaderSource

use of com.qlangtech.tis.realtime.ReaderSource in project plugins by qlangtech.

the class FlinkCDCPostgreSQLSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName dataxName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        BasicDataXRdbmsReader rdbmsReader = (BasicDataXRdbmsReader) dataSource;
        final BasicDataSourceFactory dsFactory = (BasicDataSourceFactory) rdbmsReader.getDataSourceFactory();
        BasicDataSourceFactory.ISchemaSupported schemaSupported = (BasicDataSourceFactory.ISchemaSupported) dsFactory;
        if (StringUtils.isEmpty(schemaSupported.getDBSchema())) {
            throw new IllegalStateException("dsFactory:" + dsFactory.dbName + " relevant dbSchema can not be null");
        }
        SourceChannel sourceChannel = new SourceChannel(SourceChannel.getSourceFunction(dsFactory, (tab) -> schemaSupported.getDBSchema() + "." + tab.getTabName(), tabs, (dbHost, dbs, tbs, debeziumProperties) -> {
            DateTimeConverter.setDatetimeConverters(PGDateTimeConverter.class.getName(), debeziumProperties);
            return dbs.stream().map((dbname) -> {
                SourceFunction<DTO> sourceFunction = PostgreSQLSource.<DTO>builder().hostname(dbHost).port(dsFactory.port).database(// monitor postgres database
                dbname).schemaList(// monitor inventory schema
                schemaSupported.getDBSchema()).tableList(// monitor products table
                tbs.toArray(new String[tbs.size()])).username(dsFactory.userName).password(dsFactory.password).debeziumProperties(debeziumProperties).deserializer(// converts SourceRecord to JSON String
                new TISDeserializationSchema()).build();
                return new ReaderSource(dbHost + ":" + dsFactory.port + "_" + dbname, sourceFunction);
            }).collect(Collectors.toList());
        }));
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(dataxName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}