Examples with BasicDataSourceFactory com.qlangtech.tis.plugin.ds.BasicDataSourceFactory used on opensource projects

Example 1 with BasicDataSourceFactory

use of com.qlangtech.tis.plugin.ds.BasicDataSourceFactory in project plugins by qlangtech.

the class InitWriterTable method process.

/**
 * 初始化表RDBMS的表，如果表不存在就创建表
 *
 * @param
 * @throws Exception
 */
public static void process(String dataXName, String tableName, List<String> jdbcUrls) throws Exception {
    if (StringUtils.isEmpty(dataXName)) {
        throw new IllegalArgumentException("param dataXName can not be null");
    }
    BasicDataXRdbmsWriter<BasicDataSourceFactory> dataXWriter = (BasicDataXRdbmsWriter<BasicDataSourceFactory>) DataxWriter.load(null, dataXName);
    Objects.requireNonNull(dataXWriter, "dataXWriter can not be null,dataXName:" + dataXName);
    boolean autoCreateTable = dataXWriter.autoCreateTable;
    if (autoCreateTable) {
        DataxProcessor processor = DataxProcessor.load(null, dataXName);
        File createDDL = new File(processor.getDataxCreateDDLDir(null), tableName + IDataxProcessor.DATAX_CREATE_DDL_FILE_NAME_SUFFIX);
        if (!createDDL.exists()) {
            throw new IllegalStateException("create table script is not exist:" + createDDL.getAbsolutePath());
        }
        BasicDataSourceFactory dsFactory = dataXWriter.getDataSourceFactory();
        String createScript = FileUtils.readFileToString(createDDL, TisUTF8.get());
        for (String jdbcUrl : jdbcUrls) {
            try (Connection conn = dsFactory.getConnection(jdbcUrl)) {
                List<String> tabs = Lists.newArrayList();
                dsFactory.refectTableInDB(tabs, conn);
                if (!tabs.contains(tableName)) {
                    // 表不存在
                    try (Statement statement = conn.createStatement()) {
                        logger.info("create table:{}\n   script:{}", tableName, createScript);
                        statement.execute(createScript);
                    }
                } else {
                    logger.info("table:{} already exist ,skip the create table step", tableName);
                }
            }
        }
    }
}

Example 2 with BasicDataSourceFactory

use of com.qlangtech.tis.plugin.ds.BasicDataSourceFactory in project plugins by qlangtech.

the class FlinkCDCMysqlSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName dataxName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        // TabColIndexer colIndexer = new TabColIndexer(tabs);
        // TISDeserializationSchema deserializationSchema
        // = new TISDeserializationSchema(new MySQLSourceValConvert(colIndexer));
        TISDeserializationSchema deserializationSchema = new TISDeserializationSchema();
        BasicDataXRdbmsReader rdbmsReader = (BasicDataXRdbmsReader) dataSource;
        BasicDataSourceFactory dsFactory = (BasicDataSourceFactory) rdbmsReader.getDataSourceFactory();
        SourceChannel sourceChannel = new SourceChannel(SourceChannel.getSourceFunction(dsFactory, tabs, (dbHost, dbs, tbs, debeziumProperties) -> {
            DateTimeConverter.setDatetimeConverters(MySqlDateTimeConverter.class.getName(), debeziumProperties);
            String[] databases = dbs.toArray(new String[dbs.size()]);
            return Collections.singletonList(new ReaderSource(dbHost + ":" + dsFactory.port + ":" + dbs.stream().collect(Collectors.joining("_")), MySqlSource.<DTO>builder().hostname(dbHost).port(dsFactory.port).databaseList(// monitor all tables under inventory database
            databases).tableList(tbs.toArray(new String[tbs.size()])).serverTimeZone(BasicDataSourceFactory.DEFAULT_SERVER_TIME_ZONE.getId()).username(dsFactory.getUserName()).password(dsFactory.getPassword()).startupOptions(sourceFactory.getStartupOptions()).debeziumProperties(debeziumProperties).deserializer(// converts SourceRecord to JSON String
            deserializationSchema).build()));
        }));
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(dataxName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}

Example 3 with BasicDataSourceFactory

use of com.qlangtech.tis.plugin.ds.BasicDataSourceFactory in project plugins by qlangtech.

the class TestTISFlinkCDCOracleSourceFunction method createMySqlDataSourceFactory.

protected BasicDataSourceFactory createMySqlDataSourceFactory(TargetResName dataxName) {
    Descriptor mySqlV5DataSourceFactory = TIS.get().getDescriptor("OracleDataSourceFactory");
    Assert.assertNotNull(mySqlV5DataSourceFactory);
    Descriptor.FormData formData = new Descriptor.FormData();
    formData.addProp("name", "oracle");
    formData.addProp("dbName", oracleContainer.getSid());
    formData.addProp("nodeDesc", oracleContainer.getHost());
    // formData.addProp("password", OracleTestUtils.ORACLE_PWD);
    // formData.addProp("userName", OracleTestUtils.ORACLE_USER);
    formData.addProp("password", oracleContainer.getPassword());
    formData.addProp("userName", oracleContainer.getUsername());
    formData.addProp("port", String.valueOf(oracleContainer.getOraclePort()));
    formData.addProp("allAuthorized", "true");
    formData.addProp("asServiceName", "false");
    Descriptor.ParseDescribable<BasicDataSourceFactory> parseDescribable = mySqlV5DataSourceFactory.newInstance(dataxName.getName(), formData);
    Assert.assertNotNull(parseDescribable.instance);
    return parseDescribable.instance;
}

Example 4 with BasicDataSourceFactory

use of com.qlangtech.tis.plugin.ds.BasicDataSourceFactory in project plugins by qlangtech.

the class TestFlinkCDCPostgreSQLSourceFunction method createPGDataSourceFactory.

protected BasicDataSourceFactory createPGDataSourceFactory(TargetResName dataxName) {
    Descriptor pgDataSourceFactory = TIS.get().getDescriptor("PGDataSourceFactory");
    Assert.assertNotNull(pgDataSourceFactory);
    Descriptor.FormData formData = new Descriptor.FormData();
    formData.addProp("name", "pg");
    formData.addProp("dbName", POSTGERS_CONTAINER.getDatabaseName());
    formData.addProp("nodeDesc", POSTGERS_CONTAINER.getContainerIpAddress());
    formData.addProp("password", POSTGERS_CONTAINER.getPassword());
    formData.addProp("userName", POSTGERS_CONTAINER.getUsername());
    formData.addProp("port", String.valueOf(POSTGERS_CONTAINER.getMappedPort(PostgreSQLContainer.POSTGRESQL_PORT)));
    formData.addProp("tabSchema", schemaName);
    formData.addProp("encode", "utf8");
    // formData.addProp("useCompression", "true");
    Descriptor.ParseDescribable<BasicDataSourceFactory> parseDescribable = pgDataSourceFactory.newInstance(dataxName.getName(), formData);
    Assert.assertNotNull(parseDescribable.instance);
    return parseDescribable.instance;
}

Example 5 with BasicDataSourceFactory

use of com.qlangtech.tis.plugin.ds.BasicDataSourceFactory in project plugins by qlangtech.

the class SourceChannel method getSourceFunction.

// https://ververica.github.io/flink-cdc-connectors/master/
public static List<ReaderSource> getSourceFunction(BasicDataSourceFactory dsFactory, Function<DBTable, String> tabnameCreator, List<ISelectedTab> tabs, ReaderSourceCreator sourceFunctionCreator) {
    try {
        DBConfig dbConfig = dsFactory.getDbConfig();
        List<ReaderSource> sourceFuncs = Lists.newArrayList();
        Map<String, List<String>> ip2dbs = Maps.newHashMap();
        Map<String, List<ISelectedTab>> db2tabs = Maps.newHashMap();
        dbConfig.vistDbName((config, ip, dbName) -> {
            List<String> dbs = ip2dbs.get(ip);
            if (dbs == null) {
                dbs = Lists.newArrayList();
                ip2dbs.put(ip, dbs);
            }
            dbs.add(dbName);
            if (db2tabs.get(dbName) == null) {
                db2tabs.put(dbName, tabs);
            }
            return false;
        });
        for (Map.Entry<String, List<String>> /**
         *dbs
         */
        entry : ip2dbs.entrySet()) {
            // Set<String> tbs = entry.getValue().stream().flatMap(
            // (dbName) -> db2tabs.get(dbName).stream().map((tab) -> dbName + "." + tab.getName())).collect(Collectors.toSet());
            Set<String> tbs = entry.getValue().stream().flatMap((dbName) -> db2tabs.get(dbName).stream().map((tab) -> {
                // return (dsSchemaSupport ? ((BasicDataSourceFactory.ISchemaSupported) dsFactory).getDBSchema() : dbName) + "." + tab.getName();
                return tabnameCreator.apply(new DBTable(dbName, tab));
            })).collect(Collectors.toSet());
            Properties debeziumProperties = new Properties();
            // do not use lock
            debeziumProperties.put("snapshot.locking.mode", "none");
            String dbHost = entry.getKey();
            List<String> dbs = entry.getValue();
            sourceFuncs.addAll(sourceFunctionCreator.create(dbHost, dbs, tbs, debeziumProperties));
        }
        return sourceFuncs;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}