Examples with ISelectedTab com.qlangtech.tis.plugin.ds.ISelectedTab used on opensource projects

Example 21 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project plugins by qlangtech.

the class FlinkCDCPostgreSQLSourceFunction method start.

@Override
public JobExecutionResult start(TargetResName dataxName, IDataxReader dataSource, List<ISelectedTab> tabs, IDataxProcessor dataXProcessor) throws MQConsumeException {
    try {
        BasicDataXRdbmsReader rdbmsReader = (BasicDataXRdbmsReader) dataSource;
        final BasicDataSourceFactory dsFactory = (BasicDataSourceFactory) rdbmsReader.getDataSourceFactory();
        BasicDataSourceFactory.ISchemaSupported schemaSupported = (BasicDataSourceFactory.ISchemaSupported) dsFactory;
        if (StringUtils.isEmpty(schemaSupported.getDBSchema())) {
            throw new IllegalStateException("dsFactory:" + dsFactory.dbName + " relevant dbSchema can not be null");
        }
        SourceChannel sourceChannel = new SourceChannel(SourceChannel.getSourceFunction(dsFactory, (tab) -> schemaSupported.getDBSchema() + "." + tab.getTabName(), tabs, (dbHost, dbs, tbs, debeziumProperties) -> {
            DateTimeConverter.setDatetimeConverters(PGDateTimeConverter.class.getName(), debeziumProperties);
            return dbs.stream().map((dbname) -> {
                SourceFunction<DTO> sourceFunction = PostgreSQLSource.<DTO>builder().hostname(dbHost).port(dsFactory.port).database(// monitor postgres database
                dbname).schemaList(// monitor inventory schema
                schemaSupported.getDBSchema()).tableList(// monitor products table
                tbs.toArray(new String[tbs.size()])).username(dsFactory.userName).password(dsFactory.password).debeziumProperties(debeziumProperties).deserializer(// converts SourceRecord to JSON String
                new TISDeserializationSchema()).build();
                return new ReaderSource(dbHost + ":" + dsFactory.port + "_" + dbname, sourceFunction);
            }).collect(Collectors.toList());
        }));
        for (ISelectedTab tab : tabs) {
            sourceChannel.addFocusTab(tab.getName());
        }
        return (JobExecutionResult) getConsumerHandle().consume(dataxName, sourceChannel, dataXProcessor);
    } catch (Exception e) {
        throw new MQConsumeException(e.getMessage(), e);
    }
}

Example 22 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project plugins by qlangtech.

the class TISFlinkCDCStart method deploy.

private static void deploy(TargetResName dataxName, BasicFlinkSourceHandle tableStreamHandle, ReplicasSpec incrSpec, long timestamp) throws Exception {
    // BasicFlinkSourceHandle tisFlinkSourceHandle = new TISFlinkSourceHandle();
    if (tableStreamHandle == null) {
        throw new IllegalStateException("tableStreamHandle has not been instantiated");
    }
    // ElasticSearchSinkFactory esSinkFactory = new ElasticSearchSinkFactory();
    // IPluginContext pluginContext = IPluginContext.namedContext(dataxName.getName());
    // List<TISSinkFactory> sinkFactories = TISSinkFactory.sinkFactory.getPlugins(pluginContext, null);
    // logger.info("sinkFactories size:" + sinkFactories.size());
    // for (TISSinkFactory factory : sinkFactories) {
    // sinkFactory = factory;
    // break;
    // }
    // Objects.requireNonNull(sinkFactory, "sinkFactories.size():" + sinkFactories.size());
    tableStreamHandle.setSinkFuncFactory(TISSinkFactory.getIncrSinKFactory(dataxName.getName()));
    // List<MQListenerFactory> mqFactories = HeteroEnum.MQ.getPlugins(pluginContext, null);
    MQListenerFactory mqFactory = HeteroEnum.getIncrSourceListenerFactory(dataxName.getName());
    mqFactory.setConsumerHandle(tableStreamHandle);
    // for (MQListenerFactory factory : mqFactories) {
    // factory.setConsumerHandle(tableStreamHandle);
    // mqFactory = factory;
    // }
    // Objects.requireNonNull(mqFactory, "mqFactory can not be null, mqFactories size:" + mqFactories.size());
    IMQListener mq = mqFactory.create();
    IDataxProcessor dataXProcess = DataxProcessor.load(null, dataxName.getName());
    DataxReader reader = (DataxReader) dataXProcess.getReader(null);
    if (reader == null) {
        throw new IllegalStateException("dataXReader is illegal");
    }
    // DBConfigGetter rdbmsReader = (DBConfigGetter) reader;
    List<ISelectedTab> tabs = reader.getSelectedTabs();
    mq.start(dataxName, reader, tabs, dataXProcess);
}

Example 23 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project plugins by qlangtech.

the class TestDataXDorisWriter method testDescriptorsJSONGenerate.

public void testDescriptorsJSONGenerate() {
    DataxReader dataxReader = EasyMock.createMock("dataxReader", DataxReader.class);
    List<ISelectedTab> selectedTabs = TestSelectedTabs.createSelectedTabs(1).stream().map((t) -> t).collect(Collectors.toList());
    for (ISelectedTab tab : selectedTabs) {
        for (ISelectedTab.ColMeta cm : tab.getCols()) {
            cm.setType(DataXReaderColType.STRING.dataType);
        }
    }
    // EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs).anyTimes();
    DataxReader.dataxReaderThreadLocal.set(dataxReader);
    EasyMock.replay(dataxReader);
    DataXDorisWriter writer = new DataXDorisWriter();
    DescriptorsJSON descJson = new DescriptorsJSON(writer.getDescriptor());
    JsonUtil.assertJSONEqual(DataXDorisWriter.class, "doris-datax-writer-descriptor.json", descJson.getDescriptorsJSON(), (m, e, a) -> {
        assertEquals(m, e, a);
    });
    JsonUtil.assertJSONEqual(DataXDorisWriter.class, "doris-datax-writer-descriptor.json", descJson.getDescriptorsJSON(), (m, e, a) -> {
        assertEquals(m, e, a);
    });
    EasyMock.verify(dataxReader);
}

Example 24 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project plugins by qlangtech.

the class TestDataXStarRocksWriter method testDescriptorsJSONGenerate.

public void testDescriptorsJSONGenerate() {
    DataxReader dataxReader = EasyMock.createMock("dataxReader", DataxReader.class);
    List<ISelectedTab> selectedTabs = TestSelectedTabs.createSelectedTabs(1).stream().map((t) -> t).collect(Collectors.toList());
    for (ISelectedTab tab : selectedTabs) {
        for (ISelectedTab.ColMeta cm : tab.getCols()) {
            cm.setType(DataXReaderColType.STRING.dataType);
        }
    }
    // EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs).anyTimes();
    DataxReader.dataxReaderThreadLocal.set(dataxReader);
    EasyMock.replay(dataxReader);
    DataXStarRocksWriter writer = new DataXStarRocksWriter();
    DescriptorsJSON descJson = new DescriptorsJSON(writer.getDescriptor());
    JsonUtil.assertJSONEqual(DataXStarRocksWriter.class, "starrocks-datax-writer-descriptor.json", descJson.getDescriptorsJSON(), (m, e, a) -> {
        assertEquals(m, e, a);
    });
    JsonUtil.assertJSONEqual(DataXStarRocksWriter.class, "starrocks-datax-writer-descriptor.json", descJson.getDescriptorsJSON(), (m, e, a) -> {
        assertEquals(m, e, a);
    });
    EasyMock.verify(dataxReader);
}

Example 25 with ISelectedTab

use of com.qlangtech.tis.plugin.ds.ISelectedTab in project plugins by qlangtech.

the class DataXElasticsearchWriter method initSchemaMetaContent.

@Override
public SchemaMetaContent initSchemaMetaContent(ISelectedTab tab) {
    SchemaMetaContent metaContent = new SchemaMetaContent();
    ESSchema schema = new ESSchema();
    metaContent.parseResult = schema;
    ESField field = null;
    for (ISelectedTab.ColMeta m : tab.getCols()) {
        field = new ESField();
        field.setName(m.getName());
        field.setStored(true);
        field.setIndexed(true);
        field.setType(this.mapSearchEngineType(m.getType().getCollapse()));
        // field.setSharedKey();
        // field.setUniqueKey();
        schema.fields.add(field);
    }
    byte[] schemaContent = null;
    metaContent.content = schemaContent;
    return metaContent;
}