Search in sources :

Example 1 with IDataxReader

use of com.qlangtech.tis.datax.IDataxReader in project plugins by qlangtech.

the class ClickHouseSinkFactory method createSinkFunction.

@Override
public Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> createSinkFunction(IDataxProcessor dataxProcessor) {
    Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> sinkFuncs = Maps.newHashMap();
    IDataxProcessor.TableAlias tableName = null;
    DataXClickhouseWriter dataXWriter = (DataXClickhouseWriter) dataxProcessor.getWriter(null);
    Objects.requireNonNull(dataXWriter, "dataXWriter can not be null");
    IDataxReader reader = dataxProcessor.getReader(null);
    List<ISelectedTab> tabs = reader.getSelectedTabs();
    ClickHouseDataSourceFactory dsFactory = dataXWriter.getDataSourceFactory();
    DBConfig dbConfig = dsFactory.getDbConfig();
    for (Map.Entry<String, IDataxProcessor.TableAlias> tabAliasEntry : dataxProcessor.getTabAlias().entrySet()) {
        tableName = tabAliasEntry.getValue();
        Objects.requireNonNull(tableName, "tableName can not be null");
        if (StringUtils.isEmpty(tableName.getFrom())) {
            throw new IllegalStateException("tableName.getFrom() can not be empty");
        }
        AtomicReference<SinkFunction<DTO>> sinkFuncRef = new AtomicReference<>();
        final IDataxProcessor.TableAlias tabName = tableName;
        AtomicReference<Object[]> exceptionLoader = new AtomicReference<>();
        final String targetTabName = tableName.getTo();
        dbConfig.vistDbURL(false, (dbName, jdbcUrl) -> {
            try {
                Optional<ISelectedTab> selectedTab = tabs.stream().filter((tab) -> StringUtils.equals(tabName.getFrom(), tab.getName())).findFirst();
                if (!selectedTab.isPresent()) {
                    throw new IllegalStateException("target table:" + tabName.getFrom() + " can not find matched table in:[" + tabs.stream().map((t) -> t.getName()).collect(Collectors.joining(",")) + "]");
                }
                /**
                 * 需要先初始化表starrocks目标库中的表
                 */
                dataXWriter.initWriterTable(targetTabName, Collections.singletonList(jdbcUrl));
                sinkFuncRef.set(createSinkFunction(dbName, targetTabName, selectedTab.get(), jdbcUrl, dsFactory));
            } catch (Throwable e) {
                exceptionLoader.set(new Object[] { jdbcUrl, e });
            }
        });
        if (exceptionLoader.get() != null) {
            Object[] error = exceptionLoader.get();
            throw new RuntimeException((String) error[0], (Throwable) error[1]);
        }
        Objects.requireNonNull(sinkFuncRef.get(), "sinkFunc can not be null");
        sinkFuncs.put(tableName, sinkFuncRef.get());
    }
    if (sinkFuncs.size() < 1) {
        throw new IllegalStateException("size of sinkFuncs can not be small than 1");
    }
    return sinkFuncs;
}
Also used : StringUtils(org.apache.commons.lang.StringUtils) java.util(java.util) ISelectedTab(com.qlangtech.tis.plugin.ds.ISelectedTab) FormField(com.qlangtech.tis.plugin.annotation.FormField) DTO(com.qlangtech.tis.realtime.transfer.DTO) ColumnMetaData(com.qlangtech.tis.plugin.ds.ColumnMetaData) AtomicReference(java.util.concurrent.atomic.AtomicReference) DataXClickhouseWriter(com.qlangtech.tis.plugin.datax.DataXClickhouseWriter) ClickHouseDataSourceFactory(com.qlangtech.tis.plugin.ds.clickhouse.ClickHouseDataSourceFactory) ClickHouseClusterSettings(ru.ivi.opensource.flinkclickhousesink.model.ClickHouseClusterSettings) CollectionUtils(org.apache.commons.collections.CollectionUtils) IDataXPluginMeta(com.qlangtech.tis.datax.IDataXPluginMeta) IDataxReader(com.qlangtech.tis.datax.IDataxReader) Public(org.apache.flink.annotation.Public) IDataxProcessor(com.qlangtech.tis.datax.IDataxProcessor) FormFieldType(com.qlangtech.tis.plugin.annotation.FormFieldType) Validator(com.qlangtech.tis.plugin.annotation.Validator) TISSinkFactory(com.qlangtech.tis.plugin.incr.TISSinkFactory) DBConfig(com.qlangtech.tis.plugin.ds.DBConfig) TISExtension(com.qlangtech.tis.extension.TISExtension) SinkFunction(org.apache.flink.streaming.api.functions.sink.SinkFunction) Lists(org.apache.commons.compress.utils.Lists) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) ClickHouseSinkConst(ru.ivi.opensource.flinkclickhousesink.model.ClickHouseSinkConst) ISelectedTab(com.qlangtech.tis.plugin.ds.ISelectedTab) IDataxProcessor(com.qlangtech.tis.datax.IDataxProcessor) AtomicReference(java.util.concurrent.atomic.AtomicReference) IDataxReader(com.qlangtech.tis.datax.IDataxReader) DBConfig(com.qlangtech.tis.plugin.ds.DBConfig) SinkFunction(org.apache.flink.streaming.api.functions.sink.SinkFunction) DataXClickhouseWriter(com.qlangtech.tis.plugin.datax.DataXClickhouseWriter) ClickHouseDataSourceFactory(com.qlangtech.tis.plugin.ds.clickhouse.ClickHouseDataSourceFactory)

Example 2 with IDataxReader

use of com.qlangtech.tis.datax.IDataxReader in project plugins by qlangtech.

the class StarRocksSinkFactory method createSinkFunction.

@Override
public Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> createSinkFunction(IDataxProcessor dataxProcessor) {
    Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> sinkFuncs = Maps.newHashMap();
    IDataxProcessor.TableAlias tableName = null;
    // Map<String, IDataxProcessor.TableAlias> tabAlias = dataxProcessor.getTabAlias();
    BasicDorisStarRocksWriter dataXWriter = (BasicDorisStarRocksWriter) dataxProcessor.getWriter(null);
    Objects.requireNonNull(dataXWriter, "dataXWriter can not be null");
    BasicDorisStarRocksWriter.Separator separator = dataXWriter.getSeparator();
    IDataxReader reader = dataxProcessor.getReader(null);
    List<ISelectedTab> tabs = reader.getSelectedTabs();
    DorisSourceFactory dsFactory = dataXWriter.getDataSourceFactory();
    DBConfig dbConfig = dsFactory.getDbConfig();
    Map<String, IDataxProcessor.TableAlias> selectedTabs = dataxProcessor.getTabAlias();
    if (MapUtils.isEmpty(selectedTabs)) {
        throw new IllegalStateException("selectedTabs can not be empty");
    }
    for (Map.Entry<String, IDataxProcessor.TableAlias> tabAliasEntry : selectedTabs.entrySet()) {
        tableName = tabAliasEntry.getValue();
        Objects.requireNonNull(tableName, "tableName can not be null");
        if (StringUtils.isEmpty(tableName.getFrom())) {
            throw new IllegalStateException("tableName.getFrom() can not be empty");
        }
        AtomicReference<SinkFunction<DTO>> sinkFuncRef = new AtomicReference<>();
        final IDataxProcessor.TableAlias tabName = tableName;
        AtomicReference<Object[]> exceptionLoader = new AtomicReference<>();
        final String targetTabName = tableName.getTo();
        dbConfig.vistDbURL(false, (dbName, jdbcUrl) -> {
            try {
                Optional<ISelectedTab> selectedTab = tabs.stream().filter((tab) -> StringUtils.equals(tabName.getFrom(), tab.getName())).findFirst();
                if (!selectedTab.isPresent()) {
                    throw new IllegalStateException("target table:" + tabName.getFrom() + " can not find matched table in:[" + tabs.stream().map((t) -> t.getName()).collect(Collectors.joining(",")) + "]");
                }
                /**
                 * 需要先初始化表starrocks目标库中的表
                 */
                dataXWriter.initWriterTable(targetTabName, Collections.singletonList(jdbcUrl));
                sinkFuncRef.set(createSinkFunction(dbName, targetTabName, selectedTab.get(), jdbcUrl, dsFactory, separator));
            } catch (Throwable e) {
                exceptionLoader.set(new Object[] { jdbcUrl, e });
            }
        });
        if (exceptionLoader.get() != null) {
            Object[] error = exceptionLoader.get();
            throw new RuntimeException((String) error[0], (Throwable) error[1]);
        }
        Objects.requireNonNull(sinkFuncRef.get(), "sinkFunc can not be null");
        sinkFuncs.put(tableName, sinkFuncRef.get());
    }
    if (sinkFuncs.size() < 1) {
        throw new IllegalStateException("size of sinkFuncs can not be small than 1");
    }
    return sinkFuncs;
}
Also used : StringUtils(org.apache.commons.lang.StringUtils) AtomicDataType(org.apache.flink.table.types.AtomicDataType) java.util(java.util) ISelectedTab(com.qlangtech.tis.plugin.ds.ISelectedTab) Option(com.qlangtech.tis.manage.common.Option) FormField(com.qlangtech.tis.plugin.annotation.FormField) DTO(com.qlangtech.tis.realtime.transfer.DTO) StarRocksSinkOptions(com.starrocks.connector.flink.table.StarRocksSinkOptions) AtomicReference(java.util.concurrent.atomic.AtomicReference) Context(com.alibaba.citrus.turbine.Context) IFieldErrorHandler(com.qlangtech.tis.runtime.module.misc.IFieldErrorHandler) Lists(com.google.common.collect.Lists) StarRocksSinkOP(com.starrocks.connector.flink.row.StarRocksSinkOP) TextElement(org.apache.flink.configuration.description.TextElement) IDataXPluginMeta(com.qlangtech.tis.datax.IDataXPluginMeta) IDataxReader(com.qlangtech.tis.datax.IDataxReader) org.apache.flink.table.types.logical(org.apache.flink.table.types.logical) Public(org.apache.flink.annotation.Public) ConfigOption(org.apache.flink.configuration.ConfigOption) IDataxProcessor(com.qlangtech.tis.datax.IDataxProcessor) FormFieldType(com.qlangtech.tis.plugin.annotation.FormFieldType) DorisSourceFactory(com.qlangtech.tis.plugin.ds.doris.DorisSourceFactory) Validator(com.qlangtech.tis.plugin.annotation.Validator) TISSinkFactory(com.qlangtech.tis.plugin.incr.TISSinkFactory) StarRocksSink(com.starrocks.connector.flink.StarRocksSink) DBConfig(com.qlangtech.tis.plugin.ds.DBConfig) UnderlineUtils(com.qlangtech.tis.realtime.transfer.UnderlineUtils) TISExtension(com.qlangtech.tis.extension.TISExtension) SinkFunction(org.apache.flink.streaming.api.functions.sink.SinkFunction) MapUtils(org.apache.commons.collections.MapUtils) BasicDorisStarRocksWriter(com.qlangtech.tis.plugin.datax.BasicDorisStarRocksWriter) DataTypes(org.apache.flink.table.api.DataTypes) TableSchema(org.apache.flink.table.api.TableSchema) Field(java.lang.reflect.Field) Maps(com.google.common.collect.Maps) Collectors(java.util.stream.Collectors) DataType(com.qlangtech.tis.plugin.ds.DataType) StarRocksSinkSemantic(com.starrocks.connector.flink.table.StarRocksSinkSemantic) BlockElement(org.apache.flink.configuration.description.BlockElement) ISelectedTab(com.qlangtech.tis.plugin.ds.ISelectedTab) BasicDorisStarRocksWriter(com.qlangtech.tis.plugin.datax.BasicDorisStarRocksWriter) IDataxReader(com.qlangtech.tis.datax.IDataxReader) SinkFunction(org.apache.flink.streaming.api.functions.sink.SinkFunction) IDataxProcessor(com.qlangtech.tis.datax.IDataxProcessor) AtomicReference(java.util.concurrent.atomic.AtomicReference) DBConfig(com.qlangtech.tis.plugin.ds.DBConfig) DorisSourceFactory(com.qlangtech.tis.plugin.ds.doris.DorisSourceFactory)

Example 3 with IDataxReader

use of com.qlangtech.tis.datax.IDataxReader in project plugins by qlangtech.

the class TestStarRocksSinkFactory method testStartRocksWrite.

public void testStartRocksWrite() throws Exception {
    /**
     *         CREATE TABLE `totalpayinfo` (
     *         `id` varchar(32) NULL COMMENT "",
     *         `entity_id` varchar(10) NULL COMMENT "",
     *         `num` int(11) NULL COMMENT "",
     *         `create_time` bigint(20) NULL COMMENT "",
     *         `update_time` DATETIME   NULL,
     *         `update_date` DATE       NULL,
     *         `start_time`  DATETIME   NULL
     *         ) ENGINE=OLAP
     *         UNIQUE KEY(`id`)
     *         DISTRIBUTED BY HASH(`id`) BUCKETS 10
     *         PROPERTIES (
     *         "replication_num" = "1",
     *         "in_memory" = "false",
     *         "storage_format" = "DEFAULT"
     *         );
     */
    String tableName = "totalpayinfo";
    String colEntityId = "entity_id";
    String colNum = "num";
    String colId = "id";
    String colCreateTime = "create_time";
    String updateTime = "update_time";
    String updateDate = "update_date";
    String starTime = "start_time";
    IDataxProcessor dataxProcessor = mock("dataxProcessor", IDataxProcessor.class);
    IDataxReader dataxReader = mock("dataxReader", IDataxReader.class);
    List<ISelectedTab> selectedTabs = Lists.newArrayList();
    SelectedTab totalpayinfo = mock(tableName, SelectedTab.class);
    EasyMock.expect(totalpayinfo.getName()).andReturn(tableName).times(1);
    List<ISelectedTab.ColMeta> cols = Lists.newArrayList();
    ISelectedTab.ColMeta cm = new ISelectedTab.ColMeta();
    cm.setName(colEntityId);
    cm.setType(new DataType(Types.VARCHAR, 6));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colNum);
    cm.setType(new DataType(Types.INTEGER));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colId);
    cm.setType(new DataType(Types.VARCHAR, 32));
    cm.setPk(true);
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colCreateTime);
    cm.setType(new DataType(Types.BIGINT));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(updateTime);
    cm.setType(new DataType(Types.TIMESTAMP));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(updateDate);
    cm.setType(new DataType(Types.DATE));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(starTime);
    cm.setType(new DataType(Types.TIMESTAMP));
    cols.add(cm);
    EasyMock.expect(totalpayinfo.getCols()).andReturn(cols).times(2);
    selectedTabs.add(totalpayinfo);
    EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs);
    EasyMock.expect(dataxProcessor.getReader(null)).andReturn(dataxReader);
    DataXDorisWriter dataXWriter = mock("dataXWriter", DataXDorisWriter.class);
    DorisSourceFactory sourceFactory = new DorisSourceFactory();
    sourceFactory.loadUrl = "[\"192.168.28.201:8030\"]";
    sourceFactory.userName = "root";
    sourceFactory.dbName = "tis";
    // sourceFactory.password = "";
    sourceFactory.port = 9030;
    sourceFactory.nodeDesc = "192.168.28.201";
    EasyMock.expect(dataXWriter.getDataSourceFactory()).andReturn(sourceFactory);
    dataXWriter.initWriterTable(tableName, Collections.singletonList("jdbc:mysql://192.168.28.201:9030/tis"));
    EasyMock.expect(dataxProcessor.getWriter(null)).andReturn(dataXWriter);
    StarRocksSinkFactory sinkFactory = new StarRocksSinkFactory();
    sinkFactory.columnSeparator = "x01";
    sinkFactory.rowDelimiter = "x02";
    sinkFactory.sinkSemantic = StarRocksSinkSemantic.AT_LEAST_ONCE.getName();
    sinkFactory.sinkBatchFlushInterval = 2000l;
    System.out.println("sinkFactory.sinkBatchFlushInterval:" + sinkFactory.sinkBatchFlushInterval);
    Map<String, IDataxProcessor.TableAlias> aliasMap = new HashMap<>();
    IDataxProcessor.TableAlias tab = new IDataxProcessor.TableAlias(tableName);
    aliasMap.put(tableName, tab);
    EasyMock.expect(dataxProcessor.getTabAlias()).andReturn(aliasMap);
    this.replay();
    Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> sinkFunction = sinkFactory.createSinkFunction(dataxProcessor);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    DTO d = new DTO();
    d.setEventType(DTO.EventType.DELETE);
    d.setTableName(tableName);
    Map<String, Object> after = Maps.newHashMap();
    after.put(colEntityId, "334556");
    after.put(colNum, "5");
    after.put(colId, "88888888887");
    after.put(colCreateTime, "20211113115959");
    after.put(updateTime, "2021-12-17T09:21:20Z");
    after.put(starTime, "2021-12-18 09:21:20");
    after.put(updateDate, "2021-12-9");
    d.setAfter(after);
    assertEquals(1, sinkFunction.size());
    for (Map.Entry<IDataxProcessor.TableAlias, SinkFunction<DTO>> entry : sinkFunction.entrySet()) {
        env.fromElements(new DTO[] { d }).addSink(entry.getValue());
        break;
    }
    env.execute("testJob");
    Thread.sleep(14000);
    this.verifyAll();
}
Also used : HashMap(java.util.HashMap) ISelectedTab(com.qlangtech.tis.plugin.ds.ISelectedTab) DataXDorisWriter(com.qlangtech.tis.plugin.datax.doris.DataXDorisWriter) IDataxReader(com.qlangtech.tis.datax.IDataxReader) SinkFunction(org.apache.flink.streaming.api.functions.sink.SinkFunction) ISelectedTab(com.qlangtech.tis.plugin.ds.ISelectedTab) SelectedTab(com.qlangtech.tis.plugin.datax.SelectedTab) IDataxProcessor(com.qlangtech.tis.datax.IDataxProcessor) DorisSourceFactory(com.qlangtech.tis.plugin.ds.doris.DorisSourceFactory) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) HashMap(java.util.HashMap) Map(java.util.Map) DTO(com.qlangtech.tis.realtime.transfer.DTO)

Example 4 with IDataxReader

use of com.qlangtech.tis.datax.IDataxReader in project plugins by qlangtech.

the class TestClickHouseSinkFactory method testCreateSinkFunction.

public void testCreateSinkFunction() throws Exception {
    String tableName = "totalpayinfo";
    String colEntityId = "entity_id";
    String colNum = "num";
    String colId = "id";
    String colCreateTime = "create_time";
    IDataxProcessor dataxProcessor = mock("dataxProcessor", IDataxProcessor.class);
    IDataxReader dataxReader = mock("dataxReader", IDataxReader.class);
    List<ISelectedTab> selectedTabs = Lists.newArrayList();
    SelectedTab totalpayinfo = mock(tableName, SelectedTab.class);
    EasyMock.expect(totalpayinfo.getName()).andReturn(tableName);
    List<ISelectedTab.ColMeta> cols = Lists.newArrayList();
    ISelectedTab.ColMeta cm = new ISelectedTab.ColMeta();
    cm.setName(colEntityId);
    cm.setType(new DataType(Types.VARCHAR, 6));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colNum);
    cm.setType(new DataType(Types.INTEGER));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colId);
    cm.setType(new DataType(Types.VARCHAR, 32));
    cm.setPk(true);
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colCreateTime);
    cm.setType(new DataType(Types.BIGINT));
    cols.add(cm);
    EasyMock.expect(totalpayinfo.getCols()).andReturn(cols).anyTimes();
    selectedTabs.add(totalpayinfo);
    EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs);
    EasyMock.expect(dataxProcessor.getReader(null)).andReturn(dataxReader);
    DataXClickhouseWriter dataXWriter = mock("dataXWriter", DataXClickhouseWriter.class);
    dataXWriter.initWriterTable(tableName, Collections.singletonList("jdbc:clickhouse://192.168.28.201:8123/tis"));
    ClickHouseDataSourceFactory sourceFactory = new ClickHouseDataSourceFactory();
    sourceFactory.userName = "default";
    sourceFactory.dbName = "tis";
    sourceFactory.password = "123456";
    sourceFactory.port = 8123;
    sourceFactory.nodeDesc = "192.168.28.201";
    EasyMock.expect(dataXWriter.getDataSourceFactory()).andReturn(sourceFactory);
    EasyMock.expect(dataxProcessor.getWriter(null)).andReturn(dataXWriter);
    Map<String, IDataxProcessor.TableAlias> aliasMap = new HashMap<>();
    IDataxProcessor.TableAlias tab = new IDataxProcessor.TableAlias(tableName);
    aliasMap.put(tableName, tab);
    EasyMock.expect(dataxProcessor.getTabAlias()).andReturn(aliasMap);
    this.replay();
    ClickHouseSinkFactory clickHouseSinkFactory = new ClickHouseSinkFactory();
    clickHouseSinkFactory.ignoringSendingException = true;
    clickHouseSinkFactory.maxBufferSize = 1;
    clickHouseSinkFactory.numRetries = 5;
    clickHouseSinkFactory.numWriters = 1;
    clickHouseSinkFactory.queueMaxCapacity = 1;
    clickHouseSinkFactory.timeout = 30;
    Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> sinkFuncs = clickHouseSinkFactory.createSinkFunction(dataxProcessor);
    assertTrue(sinkFuncs.size() > 0);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    DTO d = new DTO();
    d.setTableName(tableName);
    Map<String, Object> after = Maps.newHashMap();
    after.put(colEntityId, "334556");
    after.put(colNum, "5");
    after.put(colId, "123dsf124325253dsf123");
    after.put(colCreateTime, "20211113115959");
    d.setAfter(after);
    assertEquals(1, sinkFuncs.size());
    for (Map.Entry<IDataxProcessor.TableAlias, SinkFunction<DTO>> entry : sinkFuncs.entrySet()) {
        env.fromElements(new DTO[] { d }).addSink(entry.getValue()).name("clickhouse");
        break;
    }
    env.execute("testJob");
    Thread.sleep(5000);
    this.verifyAll();
}
Also used : HashMap(java.util.HashMap) ISelectedTab(com.qlangtech.tis.plugin.ds.ISelectedTab) IDataxReader(com.qlangtech.tis.datax.IDataxReader) SinkFunction(org.apache.flink.streaming.api.functions.sink.SinkFunction) ISelectedTab(com.qlangtech.tis.plugin.ds.ISelectedTab) SelectedTab(com.qlangtech.tis.plugin.datax.SelectedTab) IDataxProcessor(com.qlangtech.tis.datax.IDataxProcessor) StreamExecutionEnvironment(org.apache.flink.streaming.api.environment.StreamExecutionEnvironment) DataXClickhouseWriter(com.qlangtech.tis.plugin.datax.DataXClickhouseWriter) ClickHouseDataSourceFactory(com.qlangtech.tis.plugin.ds.clickhouse.ClickHouseDataSourceFactory) HashMap(java.util.HashMap) Map(java.util.Map) DTO(com.qlangtech.tis.realtime.transfer.DTO)

Example 5 with IDataxReader

use of com.qlangtech.tis.datax.IDataxReader in project plugins by qlangtech.

the class TestElasticSearchSinkFactory method testCreateSinkFunction.

/**
 * 参考:ElasticsearchSinkTestBase
 *
 * @throws Exception
 */
@Test
public void testCreateSinkFunction() throws Exception {
    String tableName = "totalpayinfo";
    String colEntityId = "entity_id";
    String colNum = "num";
    String colId = "id";
    String colCreateTime = "create_time";
    IDataxProcessor dataxProcessor = mock("dataxProcessor", IDataxProcessor.class);
    IDataxReader dataxReader = mock("dataxReader", IDataxReader.class);
    List<ISelectedTab> selectedTabs = Lists.newArrayList();
    SelectedTab totalpayinfo = mock(tableName, SelectedTab.class);
    EasyMock.expect(totalpayinfo.getName()).andReturn(tableName);
    List<ISelectedTab.ColMeta> cols = Lists.newArrayList();
    ISelectedTab.ColMeta cm = new ISelectedTab.ColMeta();
    cm.setName(colEntityId);
    cm.setType(new DataType(Types.VARCHAR, 6));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colNum);
    cm.setType(new DataType(Types.INTEGER));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colId);
    cm.setType(new DataType(Types.VARCHAR, 32));
    cm.setPk(true);
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colCreateTime);
    cm.setType(new DataType(Types.BIGINT));
    cols.add(cm);
    EasyMock.expect(totalpayinfo.getCols()).andReturn(cols).anyTimes();
    selectedTabs.add(totalpayinfo);
    EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs);
    EasyMock.expect(dataxProcessor.getReader(null)).andReturn(dataxReader);
    DataXElasticsearchWriter dataXWriter = mock("dataXWriter", DataXElasticsearchWriter.class);
    ESTableAlias esTableAlias = new ESTableAlias();
    dataXWriter.initialIndex(esTableAlias);
    EasyMock.expect(dataxProcessor.getWriter(null)).andReturn(dataXWriter);
    Map<String, IDataxProcessor.TableAlias> aliasMap = new HashMap<>();
    IDataxProcessor.TableAlias tab = new IDataxProcessor.TableAlias(tableName);
    aliasMap.put(tableName, tab);
    EasyMock.expect(dataxProcessor.getTabAlias()).andReturn(aliasMap);
    this.replay();
    ElasticSearchSinkFactory clickHouseSinkFactory = new ElasticSearchSinkFactory();
    Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> sinkFuncs = clickHouseSinkFactory.createSinkFunction(dataxProcessor);
    Assert.assertTrue("sinkFuncs must > 0", sinkFuncs.size() > 0);
    // StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    // env.setParallelism(1);
    DTO d = new DTO();
    d.setTableName(tableName);
    d.setEventType(DTO.EventType.ADD);
    Map<String, Object> after = Maps.newHashMap();
    after.put(colEntityId, "334556");
    after.put(colNum, "5");
    after.put(colId, "123dsf124325253dsf123");
    after.put(colCreateTime, "20211113115959");
    d.setAfter(after);
    Assert.assertEquals(1, sinkFuncs.size());
    for (Map.Entry<IDataxProcessor.TableAlias, SinkFunction<DTO>> entry : sinkFuncs.entrySet()) {
        // env.fromElements(new DTO[]{d}).addSink(entry.getValue()).name("clickhouse");
        runElasticSearchSinkTest("elasticsearch-sink-test-json-index", entry.getValue());
        break;
    }
    // env.execute("testJob");
    Thread.sleep(5000);
    this.verifyAll();
    Client client = getClient();
}
Also used : HashMap(java.util.HashMap) ISelectedTab(com.qlangtech.tis.plugin.ds.ISelectedTab) DataXElasticsearchWriter(com.qlangtech.tis.plugin.datax.DataXElasticsearchWriter) IDataxReader(com.qlangtech.tis.datax.IDataxReader) SinkFunction(org.apache.flink.streaming.api.functions.sink.SinkFunction) ISelectedTab(com.qlangtech.tis.plugin.ds.ISelectedTab) SelectedTab(com.qlangtech.tis.plugin.datax.SelectedTab) DataType(com.qlangtech.tis.plugin.ds.DataType) Client(org.elasticsearch.client.Client) IDataxProcessor(com.qlangtech.tis.datax.IDataxProcessor) ESTableAlias(com.qlangtech.tis.datax.impl.ESTableAlias) ESTableAlias(com.qlangtech.tis.datax.impl.ESTableAlias) HashMap(java.util.HashMap) Map(java.util.Map) DTO(com.qlangtech.tis.realtime.transfer.DTO) Test(org.junit.Test)

Aggregations

IDataxReader (com.qlangtech.tis.datax.IDataxReader)12 IDataxProcessor (com.qlangtech.tis.datax.IDataxProcessor)11 ISelectedTab (com.qlangtech.tis.plugin.ds.ISelectedTab)11 DTO (com.qlangtech.tis.realtime.transfer.DTO)10 Collectors (java.util.stream.Collectors)8 SinkFunction (org.apache.flink.streaming.api.functions.sink.SinkFunction)7 StringUtils (org.apache.commons.lang.StringUtils)6 java.util (java.util)5 Maps (com.google.common.collect.Maps)4 IDataXPluginMeta (com.qlangtech.tis.datax.IDataXPluginMeta)4 TISExtension (com.qlangtech.tis.extension.TISExtension)4 FormField (com.qlangtech.tis.plugin.annotation.FormField)4 FormFieldType (com.qlangtech.tis.plugin.annotation.FormFieldType)4 Validator (com.qlangtech.tis.plugin.annotation.Validator)4 TISSinkFactory (com.qlangtech.tis.plugin.incr.TISSinkFactory)4 Public (org.apache.flink.annotation.Public)4 Lists (com.google.common.collect.Lists)3 SourceChannel (com.qlangtech.plugins.incr.flink.cdc.SourceChannel)3 TISDeserializationSchema (com.qlangtech.plugins.incr.flink.cdc.TISDeserializationSchema)3 IAsyncMsgDeserialize (com.qlangtech.tis.async.message.client.consumer.IAsyncMsgDeserialize)3