Examples with ISelectedTab com.qlangtech.tis.plugin.ds.ISelectedTab used on opensource projects

Example 16 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project plugins by qlangtech.

the class TestFlinkCDCMongoDBSourceFunction method testStart.

public void testStart() throws Exception {
    FlinkCDCMongoDBSourceFactory mongoDBSourceFactory = this.mock("mongoDBSourceFactory", FlinkCDCMongoDBSourceFactory.class);
    FlinkCDCMongoDBSourceFunction mongoDBSourceFunction = new FlinkCDCMongoDBSourceFunction(mongoDBSourceFactory);
    DataXMongodbReader mongodbReader = new DataXMongodbReader();
    List<ISelectedTab> tabs = Lists.newArrayList();
    IDataxProcessor dataXProcessor = this.mock("dataxProcess", IDataxProcessor.class);
    this.replay();
    TargetResName dataXName = new TargetResName("test");
    mongoDBSourceFunction.start(dataXName, mongodbReader, tabs, dataXProcessor);
    this.verifyAll();
}

Example 17 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project plugins by qlangtech.

the class ElasticSearchSinkFactory method createSinkFunction.

@Override
public Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> createSinkFunction(IDataxProcessor dataxProcessor) {
    DataXElasticsearchWriter dataXWriter = (DataXElasticsearchWriter) dataxProcessor.getWriter(null);
    Objects.requireNonNull(dataXWriter, "dataXWriter can not be null");
    IHttpToken token = dataXWriter.getToken();
    ESTableAlias esSchema = null;
    for (Map.Entry<String, IDataxProcessor.TableAlias> e : dataxProcessor.getTabAlias().entrySet()) {
        IDataxProcessor.TableAlias value = e.getValue();
        if (!(value instanceof ESTableAlias)) {
            throw new IllegalStateException("value must be type of 'ESTableAlias',but now is :" + value.getClass());
        }
        esSchema = (ESTableAlias) value;
        break;
    }
    Objects.requireNonNull(esSchema, "esSchema can not be null");
    List<ISelectedTab.ColMeta> cols = esSchema.getSourceCols();
    if (CollectionUtils.isEmpty(cols)) {
        throw new IllegalStateException("cols can not be null");
    }
    Optional<ISelectedTab.ColMeta> firstPK = cols.stream().filter((c) -> c.isPk()).findFirst();
    if (!firstPK.isPresent()) {
        throw new IllegalStateException("has not set PK col");
    }
    /**
     ******************************************************
     * 初始化索引Schema
     ******************************************************
     */
    dataXWriter.initialIndex(esSchema);
    // JSONArray schemaCols = esSchema.getSchemaCols();
    // ESClient esClient = new ESClient();
    // esClient.createClient(token.getEndpoint(),
    // token.getAccessKeyId(),
    // token.getAccessKeySecret(),
    // false,
    // 300000,
    // false,
    // false);
    // try {
    // esClient.createIndex(dataXWriter.getIndexName()
    // , dataXWriter.type
    // , esClient.genMappings(schemaCols, dataXWriter.type, (columnList) -> {
    // }), dataXWriter.settings, false);
    // } catch (Exception e) {
    // throw new RuntimeException(e);
    // } finally {
    // try {
    // esClient.closeJestClient();
    // } catch (Throwable e) {
    // 
    // }
    // }
    // if (!) {
    // throw new IllegalStateException("create index or mapping failed indexName:" + dataXWriter.getIndexName());
    // }
    // Map<String, String> config = new HashMap<>();
    // config.put("cluster.name", "my-cluster-name");
    // // This instructs the sink to emit after every element, otherwise they would be buffered
    // config.put("bulk.flush.max.actions", "1");
    List<HttpHost> transportAddresses = new ArrayList<>();
    transportAddresses.add(HttpHost.create(token.getEndpoint()));
    ElasticsearchSink.Builder<DTO> sinkBuilder = new ElasticsearchSink.Builder<>(transportAddresses, new DefaultElasticsearchSinkFunction(cols.stream().map((c) -> c.getName()).collect(Collectors.toSet()), firstPK.get().getName(), dataXWriter.getIndexName()));
    if (this.bulkFlushMaxActions != null) {
        sinkBuilder.setBulkFlushMaxActions(this.bulkFlushMaxActions);
    }
    if (this.bulkFlushMaxSizeMb != null) {
        sinkBuilder.setBulkFlushMaxSizeMb(bulkFlushMaxSizeMb);
    }
    if (this.bulkFlushIntervalMs != null) {
        sinkBuilder.setBulkFlushInterval(this.bulkFlushIntervalMs);
    }
    // new RestClientBuilder.HttpClientConfigCallback() {
    // @Override
    // public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpClientBuilder) {
    // return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
    // }
    // }
    sinkBuilder.setFailureHandler(new DefaultActionRequestFailureHandler());
    if (StringUtils.isNotEmpty(token.getAccessKeyId()) || StringUtils.isNotEmpty(token.getAccessKeySecret())) {
        // 如果用户设置了accessKey 或者accessSecret
        sinkBuilder.setRestClientFactory(new TISElasticRestClientFactory(token.getAccessKeyId(), token.getAccessKeySecret()));
    // sinkBuilder.setRestClientFactory(new RestClientFactory() {
    // @Override
    // public void configureRestClientBuilder(RestClientBuilder restClientBuilder) {
    // final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
    // credentialsProvider.setCredentials(AuthScope.ANY,
    // new UsernamePasswordCredentials(token.getAccessKeyId(), token.getAccessKeySecret()));
    // restClientBuilder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() {
    // @Override
    // public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) {
    // return httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
    // }
    // });
    // }
    // });
    }
    IDataxProcessor.TableAlias tableMapper = new IDataxProcessor.TableAlias();
    tableMapper.setTo(dataXWriter.getIndexName());
    IDataxReader reader = dataxProcessor.getReader(null);
    for (ISelectedTab selectedTab : reader.getSelectedTabs()) {
        tableMapper.setFrom(selectedTab.getName());
    }
    return Collections.singletonMap(tableMapper, sinkBuilder.build());
}

Example 18 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project plugins by qlangtech.

the class SourceChannel method getSourceFunction.

// https://ververica.github.io/flink-cdc-connectors/master/
public static List<ReaderSource> getSourceFunction(BasicDataSourceFactory dsFactory, Function<DBTable, String> tabnameCreator, List<ISelectedTab> tabs, ReaderSourceCreator sourceFunctionCreator) {
    try {
        DBConfig dbConfig = dsFactory.getDbConfig();
        List<ReaderSource> sourceFuncs = Lists.newArrayList();
        Map<String, List<String>> ip2dbs = Maps.newHashMap();
        Map<String, List<ISelectedTab>> db2tabs = Maps.newHashMap();
        dbConfig.vistDbName((config, ip, dbName) -> {
            List<String> dbs = ip2dbs.get(ip);
            if (dbs == null) {
                dbs = Lists.newArrayList();
                ip2dbs.put(ip, dbs);
            }
            dbs.add(dbName);
            if (db2tabs.get(dbName) == null) {
                db2tabs.put(dbName, tabs);
            }
            return false;
        });
        for (Map.Entry<String, List<String>> /**
         *dbs
         */
        entry : ip2dbs.entrySet()) {
            // Set<String> tbs = entry.getValue().stream().flatMap(
            // (dbName) -> db2tabs.get(dbName).stream().map((tab) -> dbName + "." + tab.getName())).collect(Collectors.toSet());
            Set<String> tbs = entry.getValue().stream().flatMap((dbName) -> db2tabs.get(dbName).stream().map((tab) -> {
                // return (dsSchemaSupport ? ((BasicDataSourceFactory.ISchemaSupported) dsFactory).getDBSchema() : dbName) + "." + tab.getName();
                return tabnameCreator.apply(new DBTable(dbName, tab));
            })).collect(Collectors.toSet());
            Properties debeziumProperties = new Properties();
            // do not use lock
            debeziumProperties.put("snapshot.locking.mode", "none");
            String dbHost = entry.getKey();
            List<String> dbs = entry.getValue();
            sourceFuncs.addAll(sourceFunctionCreator.create(dbHost, dbs, tbs, debeziumProperties));
        }
        return sourceFuncs;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 19 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project plugins by qlangtech.

the class TestFlinkCDCMySQLSourceFactory method testBinlogConsumeWithDataStreamRegisterInstaneDetailTable.

/**
 * 测试 instancedetail
 *
 * @throws Exception
 */
@Test
public void testBinlogConsumeWithDataStreamRegisterInstaneDetailTable() throws Exception {
    FlinkCDCMySQLSourceFactory mysqlCDCFactory = new FlinkCDCMySQLSourceFactory();
    mysqlCDCFactory.startupOptions = "latest";
    final String tabName = "instancedetail";
    CUDCDCTestSuit cdcTestSuit = new CUDCDCTestSuit() {

        @Override
        protected BasicDataSourceFactory createDataSourceFactory(TargetResName dataxName) {
            return createMySqlDataSourceFactory(dataxName);
        }

        @Override
        protected String getColEscape() {
            return "`";
        }

        @Override
        protected IResultRows createConsumerHandle(String tabName) {
            return new TestTableRegisterFlinkSourceHandle(tabName, cols);
        }

        @Override
        protected void verfiyTableCrudProcess(String tabName, BasicDataXRdbmsReader dataxReader, ISelectedTab tab, IResultRows consumerHandle, IMQListener<JobExecutionResult> imqListener) throws MQConsumeException, InterruptedException {
            // super.verfiyTableCrudProcess(tabName, dataxReader, tab, consumerHandle, imqListener);
            List<ISelectedTab> tabs = Collections.singletonList(tab);
            List<TestRow> exampleRows = Lists.newArrayList();
            exampleRows.add(this.parseTestRow(RowKind.INSERT, TestFlinkCDCMySQLSourceFactory.class, tabName + "/insert1.txt"));
            Assert.assertEquals(1, exampleRows.size());
            imqListener.start(dataxName, dataxReader, tabs, null);
            Thread.sleep(1000);
            CloseableIterator<Row> snapshot = consumerHandle.getRowSnapshot(tabName);
            BasicDataSourceFactory dataSourceFactory = (BasicDataSourceFactory) dataxReader.getDataSourceFactory();
            Assert.assertNotNull("dataSourceFactory can not be null", dataSourceFactory);
            dataSourceFactory.visitFirstConnection((conn) -> {
                startProcessConn(conn);
                for (TestRow t : exampleRows) {
                    RowVals<Object> vals = t.vals;
                    final String insertBase = "insert into " + createTableName(tabName) + "(" + cols.stream().filter((c) -> vals.notNull(c.getName())).map((col) -> getColEscape() + col.getName() + getColEscape()).collect(Collectors.joining(" , ")) + ") " + "values(" + cols.stream().filter((c) -> vals.notNull(c.getName())).map((col) -> "?").collect(Collectors.joining(" , ")) + ")";
                    PreparedStatement statement = conn.prepareStatement(insertBase);
                    AtomicInteger ci = new AtomicInteger();
                    cols.stream().filter((c) -> vals.notNull(c.getName())).forEach((col) -> {
                        col.type.accept(new DataType.TypeVisitor<Void>() {

                            @Override
                            public Void longType(DataType type) {
                                try {
                                    statement.setLong(ci.incrementAndGet(), Long.parseLong(vals.getString(col.getName())));
                                } catch (SQLException e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void doubleType(DataType type) {
                                try {
                                    statement.setDouble(ci.incrementAndGet(), Double.parseDouble(vals.getString(col.getName())));
                                } catch (SQLException e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void dateType(DataType type) {
                                try {
                                    statement.setDate(ci.incrementAndGet(), java.sql.Date.valueOf(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void timestampType(DataType type) {
                                try {
                                    statement.setTimestamp(ci.incrementAndGet(), java.sql.Timestamp.valueOf(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void bitType(DataType type) {
                                try {
                                    statement.setByte(ci.incrementAndGet(), Byte.parseByte(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void blobType(DataType type) {
                                try {
                                    try (InputStream input = new ByteArrayInputStream(vals.getString(col.getName()).getBytes(TisUTF8.get()))) {
                                        statement.setBlob(ci.incrementAndGet(), input);
                                    }
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void varcharType(DataType type) {
                                try {
                                    statement.setString(ci.incrementAndGet(), (vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void intType(DataType type) {
                                try {
                                    statement.setInt(ci.incrementAndGet(), Integer.parseInt(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void floatType(DataType type) {
                                try {
                                    statement.setFloat(ci.incrementAndGet(), Float.parseFloat(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void decimalType(DataType type) {
                                try {
                                    statement.setBigDecimal(ci.incrementAndGet(), BigDecimal.valueOf(Double.parseDouble(vals.getString(col.getName()))));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void timeType(DataType type) {
                                try {
                                    statement.setTime(ci.incrementAndGet(), java.sql.Time.valueOf(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void tinyIntType(DataType dataType) {
                                try {
                                    statement.setShort(ci.incrementAndGet(), Short.parseShort(vals.getString(col.getName())));
                                } catch (Exception e) {
                                    throw new RuntimeException(e);
                                }
                                return null;
                            }

                            @Override
                            public Void smallIntType(DataType dataType) {
                                tinyIntType(dataType);
                                return null;
                            }
                        });
                    });
                    Assert.assertEquals(1, executePreparedStatement(conn, statement));
                    statement.close();
                    sleepForAWhile();
                    System.out.println("wait to show insert rows");
                    waitForSnapshotStarted(snapshot);
                    List<TestRow> rows = fetchRows(snapshot, 1, false);
                    for (TestRow rr : rows) {
                        System.out.println("------------" + rr.get("instance_id"));
                        assertTestRow(tabName, RowKind.INSERT, consumerHandle, t, rr);
                    }
                }
            });
        }
    };
    cdcTestSuit.startTest(mysqlCDCFactory, tabName);
}

Example 20 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project plugins by qlangtech.

the class FlinkCDCOracleSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName channalName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        BasicDataXRdbmsReader reader = (BasicDataXRdbmsReader) dataSource;
        BasicDataSourceFactory f = (BasicDataSourceFactory) reader.getDataSourceFactory();
        SourceChannel sourceChannel = new SourceChannel(SourceChannel.getSourceFunction(f, (tab) -> tab.getTabName(), tabs, (dbHost, dbs, tbs, debeziumProperties) -> {
            return dbs.stream().map((databaseName) -> {
                SourceFunction<DTO> sourceFunction = OracleSource.<DTO>builder().hostname(dbHost).debeziumProperties(debeziumProperties).port(f.port).startupOptions(sourceFactory.getStartupOptions()).database(// monitor XE database
                StringUtils.upperCase(f.dbName)).tableList(// monitor products table
                tbs.toArray(new String[tbs.size()])).username(f.getUserName()).password(f.getPassword()).deserializer(// converts SourceRecord to JSON String
                new TISDeserializationSchema()).build();
                return new ReaderSource(dbHost + ":" + f.port + "_" + databaseName, sourceFunction);
            }).collect(Collectors.toList());
        }));
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(channalName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}