Examples with BasicDataXRdbmsReader com.qlangtech.tis.plugin.datax.common.BasicDataXRdbmsReader used on opensource projects

Example 6 with BasicDataXRdbmsReader

use of com.qlangtech.tis.plugin.datax.common.BasicDataXRdbmsReader in project plugins by qlangtech.

the class FlinkCDCPostgreSQLSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName dataxName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        BasicDataXRdbmsReader rdbmsReader = (BasicDataXRdbmsReader) dataSource;
        final BasicDataSourceFactory dsFactory = (BasicDataSourceFactory) rdbmsReader.getDataSourceFactory();
        BasicDataSourceFactory.ISchemaSupported schemaSupported = (BasicDataSourceFactory.ISchemaSupported) dsFactory;
        if (StringUtils.isEmpty(schemaSupported.getDBSchema())) {
            throw new IllegalStateException("dsFactory:" + dsFactory.dbName + " relevant dbSchema can not be null");
        }
        SourceChannel sourceChannel = new SourceChannel(SourceChannel.getSourceFunction(dsFactory, (tab) -> schemaSupported.getDBSchema() + "." + tab.getTabName(), tabs, (dbHost, dbs, tbs, debeziumProperties) -> {
            DateTimeConverter.setDatetimeConverters(PGDateTimeConverter.class.getName(), debeziumProperties);
            return dbs.stream().map((dbname) -> {
                SourceFunction<DTO> sourceFunction = PostgreSQLSource.<DTO>builder().hostname(dbHost).port(dsFactory.port).database(// monitor postgres database
                dbname).schemaList(// monitor inventory schema
                schemaSupported.getDBSchema()).tableList(// monitor products table
                tbs.toArray(new String[tbs.size()])).username(dsFactory.userName).password(dsFactory.password).debeziumProperties(debeziumProperties).deserializer(// converts SourceRecord to JSON String
                new TISDeserializationSchema()).build();
                return new ReaderSource(dbHost + ":" + dsFactory.port + "_" + dbname, sourceFunction);
            }).collect(Collectors.toList());
        }));
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(dataxName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}

Example 7 with BasicDataXRdbmsReader

use of com.qlangtech.tis.plugin.datax.common.BasicDataXRdbmsReader in project plugins by qlangtech.

the class CUDCDCTestSuit method startTest.

public void startTest(MQListenerFactory cdcFactory, String tabName) throws Exception {
    BasicDataXRdbmsReader dataxReader = createDataxReader(dataxName, tabName);
    // replay();
    List<SelectedTab> selectedTabs = dataxReader.getSelectedTabs();
    Optional<SelectedTab> firstSelectedTab = selectedTabs.stream().filter((t) -> tabName.equals(t.name)).findFirst();
    Assert.assertTrue("firstSelectedTab:" + tabName + " must be present", firstSelectedTab.isPresent());
    ISelectedTab tab = firstSelectedTab.get();
    this.cols = Lists.newArrayList();
    HdfsColMeta cMeta = null;
    for (ISelectedTab.ColMeta c : tab.getCols()) {
        cMeta = new HdfsColMeta(c.getName(), c.isNullable(), c.isPk(), c.getType());
        cols.add(cMeta);
    }
    IResultRows consumerHandle = getTestBasicFlinkSourceHandle(tabName);
    cdcFactory.setConsumerHandle(consumerHandle.getConsumerHandle());
    IMQListener<JobExecutionResult> imqListener = cdcFactory.create();
    this.verfiyTableCrudProcess(tabName, dataxReader, tab, consumerHandle, imqListener);
    consumerHandle.cancel();
}