Examples with DBConfig com.qlangtech.tis.plugin.ds.DBConfig used on opensource projects

Example 6 with DBConfig

use of com.qlangtech.tis.plugin.ds.DBConfig in project tis by qlangtech.

the class TestDBConfigParser method testJustParseDBHost.

public void testJustParseDBHost() throws Exception {
    DBConfigParser p = getDBParser("order_db_config.txt");
    DBConfig db = p.startParser();
    String dbName = "order";
    Map<String, List<String>> dbEnum = DBConfigParser.parseDBEnum(dbName, db.getHostDesc().toString());
    assertTrue(dbEnum.size() > 1);
    validateDbEnum(dbName, dbEnum);
}

Example 7 with DBConfig

use of com.qlangtech.tis.plugin.ds.DBConfig in project tis by qlangtech.

the class TestDBConfigParser method testSingle.

public void testSingle() throws Exception {
    DBConfigParser parser = getDBParser("host_desc_single.txt");
    Assert.assertTrue(parser.parseHostDesc());
    assertEquals("127.0.0.3", parser.hostDesc.toString());
    DBConfig db = parser.dbConfigResult;
    assertEquals(1, db.getDbEnum().entrySet().size());
    // StringBuffer dbdesc = new StringBuffer();
    for (Map.Entry<String, List<String>> e : db.getDbEnum().entrySet()) {
        // dbdesc.append(e.getKey()).append(":");
        // 
        // dbdesc.append("\n");
        assertEquals("127.0.0.3", e.getKey());
        assertEquals(1, e.getValue().size());
        for (String dbName : e.getValue()) {
            Assert.assertNull(dbName);
        }
    }
// System.out.println(dbdesc.toString());
}

Example 8 with DBConfig

use of com.qlangtech.tis.plugin.ds.DBConfig in project tis by qlangtech.

the class TestDBConfigParser method testMulti2.

public void testMulti2() throws Exception {
    DBConfigParser parser = getDBParser("order_db_config.txt");
    DBConfig db = parser.startParser();
    assertEquals("10.1.6.99[1-32],10.1.6.101[01-32],10.1.6.102[33-64],10.1.6.103[65-96],10.1.6.104[97-128],127.0.0.3", db.getHostDesc().toString());
    final String orderName = "order";
    Assert.assertEquals("mysql", db.getDbType());
    Assert.assertEquals(orderName, db.getName());
    Map<String, List<String>> dbEnum = db.getDbEnum();
    validateDbEnum(orderName, dbEnum);
}

Example 9 with DBConfig

use of com.qlangtech.tis.plugin.ds.DBConfig in project plugins by qlangtech.

the class ClickHouseSinkFactory method createSinkFunction.

private SinkFunction<DTO> createSinkFunction(String dbName, final String targetTabName, ISelectedTab tab, String jdbcUrl, ClickHouseDataSourceFactory dsFactory) {
    // create props for sink
    Properties props = new Properties();
    props.put(ClickHouseSinkConst.TARGET_TABLE_NAME, targetTabName);
    Objects.requireNonNull(maxBufferSize, "maxBufferSize can not be null");
    props.put(ClickHouseSinkConst.MAX_BUFFER_SIZE, String.valueOf(maxBufferSize));
    Map<String, String> globalParams = Maps.newHashMap();
    // ClickHouse cluster properties
    // "http://" + clickHouse.getContainerIpAddress() + ":" + dockerActualPort
    List<String> clickhouseHosts = Lists.newArrayList();
    DBConfig dbCfg = dsFactory.getDbConfig();
    try {
        dbCfg.vistDbName((cfg, ip, _dbName) -> {
            clickhouseHosts.add("http://" + ip + ":" + dsFactory.port + "/?database=" + _dbName);
            return false;
        });
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    if (clickhouseHosts.size() < 1) {
        throw new IllegalStateException("clickhouseHosts size can not small than 1");
    }
    globalParams.put(ClickHouseClusterSettings.CLICKHOUSE_HOSTS, clickhouseHosts.stream().collect(Collectors.joining(",")));
    globalParams.put(ClickHouseClusterSettings.CLICKHOUSE_USER, dsFactory.getUserName());
    globalParams.put(ClickHouseClusterSettings.CLICKHOUSE_PASSWORD, StringUtils.trimToEmpty(dsFactory.getPassword()));
    // sink common
    globalParams.put(ClickHouseSinkConst.TIMEOUT_SEC, String.valueOf(this.timeout));
    globalParams.put(ClickHouseSinkConst.FAILED_RECORDS_PATH, "/tmp/tis-clickhouse-sink");
    globalParams.put(ClickHouseSinkConst.NUM_WRITERS, String.valueOf(this.numWriters));
    globalParams.put(ClickHouseSinkConst.NUM_RETRIES, String.valueOf(this.numRetries));
    globalParams.put(ClickHouseSinkConst.QUEUE_MAX_CAPACITY, String.valueOf(this.queueMaxCapacity));
    globalParams.put(ClickHouseSinkConst.IGNORING_CLICKHOUSE_SENDING_EXCEPTION_ENABLED, String.valueOf(this.ignoringSendingException));
    List<ColumnMetaData> colsMeta = dsFactory.getTableMetadata(targetTabName);
    if (CollectionUtils.isEmpty(colsMeta)) {
        throw new IllegalStateException("targetTabName relevant colsMeta can not be empty");
    }
    return new TISClickHouseSink(globalParams, props, colsMeta.stream().map((c) -> new ColMeta(c.getKey(), c.getType())).collect(Collectors.toList()));
}

Example 10 with DBConfig

use of com.qlangtech.tis.plugin.ds.DBConfig in project plugins by qlangtech.

the class SourceChannel method getSourceFunction.

// https://ververica.github.io/flink-cdc-connectors/master/
public static List<ReaderSource> getSourceFunction(BasicDataSourceFactory dsFactory, Function<DBTable, String> tabnameCreator, List<ISelectedTab> tabs, ReaderSourceCreator sourceFunctionCreator) {
    try {
        DBConfig dbConfig = dsFactory.getDbConfig();
        List<ReaderSource> sourceFuncs = Lists.newArrayList();
        Map<String, List<String>> ip2dbs = Maps.newHashMap();
        Map<String, List<ISelectedTab>> db2tabs = Maps.newHashMap();
        dbConfig.vistDbName((config, ip, dbName) -> {
            List<String> dbs = ip2dbs.get(ip);
            if (dbs == null) {
                dbs = Lists.newArrayList();
                ip2dbs.put(ip, dbs);
            }
            dbs.add(dbName);
            if (db2tabs.get(dbName) == null) {
                db2tabs.put(dbName, tabs);
            }
            return false;
        });
        for (Map.Entry<String, List<String>> /**
         *dbs
         */
        entry : ip2dbs.entrySet()) {
            // Set<String> tbs = entry.getValue().stream().flatMap(
            // (dbName) -> db2tabs.get(dbName).stream().map((tab) -> dbName + "." + tab.getName())).collect(Collectors.toSet());
            Set<String> tbs = entry.getValue().stream().flatMap((dbName) -> db2tabs.get(dbName).stream().map((tab) -> {
                // return (dsSchemaSupport ? ((BasicDataSourceFactory.ISchemaSupported) dsFactory).getDBSchema() : dbName) + "." + tab.getName();
                return tabnameCreator.apply(new DBTable(dbName, tab));
            })).collect(Collectors.toSet());
            Properties debeziumProperties = new Properties();
            // do not use lock
            debeziumProperties.put("snapshot.locking.mode", "none");
            String dbHost = entry.getKey();
            List<String> dbs = entry.getValue();
            sourceFuncs.addAll(sourceFunctionCreator.create(dbHost, dbs, tbs, debeziumProperties));
        }
        return sourceFuncs;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}