Examples with ClickHouseDataSourceFactory com.qlangtech.tis.plugin.ds.clickhouse.ClickHouseDataSourceFactory used on opensource projects

Example 1 with ClickHouseDataSourceFactory

use of com.qlangtech.tis.plugin.ds.clickhouse.ClickHouseDataSourceFactory in project plugins by qlangtech.

the class ClickHouseSinkFactory method createSinkFunction.

@Override
public Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> createSinkFunction(IDataxProcessor dataxProcessor) {
    Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> sinkFuncs = Maps.newHashMap();
    IDataxProcessor.TableAlias tableName = null;
    DataXClickhouseWriter dataXWriter = (DataXClickhouseWriter) dataxProcessor.getWriter(null);
    Objects.requireNonNull(dataXWriter, "dataXWriter can not be null");
    IDataxReader reader = dataxProcessor.getReader(null);
    List<ISelectedTab> tabs = reader.getSelectedTabs();
    ClickHouseDataSourceFactory dsFactory = dataXWriter.getDataSourceFactory();
    DBConfig dbConfig = dsFactory.getDbConfig();
    for (Map.Entry<String, IDataxProcessor.TableAlias> tabAliasEntry : dataxProcessor.getTabAlias().entrySet()) {
        tableName = tabAliasEntry.getValue();
        Objects.requireNonNull(tableName, "tableName can not be null");
        if (StringUtils.isEmpty(tableName.getFrom())) {
            throw new IllegalStateException("tableName.getFrom() can not be empty");
        }
        AtomicReference<SinkFunction<DTO>> sinkFuncRef = new AtomicReference<>();
        final IDataxProcessor.TableAlias tabName = tableName;
        AtomicReference<Object[]> exceptionLoader = new AtomicReference<>();
        final String targetTabName = tableName.getTo();
        dbConfig.vistDbURL(false, (dbName, jdbcUrl) -> {
            try {
                Optional<ISelectedTab> selectedTab = tabs.stream().filter((tab) -> StringUtils.equals(tabName.getFrom(), tab.getName())).findFirst();
                if (!selectedTab.isPresent()) {
                    throw new IllegalStateException("target table:" + tabName.getFrom() + " can not find matched table in:[" + tabs.stream().map((t) -> t.getName()).collect(Collectors.joining(",")) + "]");
                }
                /**
                 * 需要先初始化表starrocks目标库中的表
                 */
                dataXWriter.initWriterTable(targetTabName, Collections.singletonList(jdbcUrl));
                sinkFuncRef.set(createSinkFunction(dbName, targetTabName, selectedTab.get(), jdbcUrl, dsFactory));
            } catch (Throwable e) {
                exceptionLoader.set(new Object[] { jdbcUrl, e });
            }
        });
        if (exceptionLoader.get() != null) {
            Object[] error = exceptionLoader.get();
            throw new RuntimeException((String) error[0], (Throwable) error[1]);
        }
        Objects.requireNonNull(sinkFuncRef.get(), "sinkFunc can not be null");
        sinkFuncs.put(tableName, sinkFuncRef.get());
    }
    if (sinkFuncs.size() < 1) {
        throw new IllegalStateException("size of sinkFuncs can not be small than 1");
    }
    return sinkFuncs;
}

Example 2 with ClickHouseDataSourceFactory

use of com.qlangtech.tis.plugin.ds.clickhouse.ClickHouseDataSourceFactory in project plugins by qlangtech.

the class TestClickHouseSinkFactory method testCreateSinkFunction.

public void testCreateSinkFunction() throws Exception {
    String tableName = "totalpayinfo";
    String colEntityId = "entity_id";
    String colNum = "num";
    String colId = "id";
    String colCreateTime = "create_time";
    IDataxProcessor dataxProcessor = mock("dataxProcessor", IDataxProcessor.class);
    IDataxReader dataxReader = mock("dataxReader", IDataxReader.class);
    List<ISelectedTab> selectedTabs = Lists.newArrayList();
    SelectedTab totalpayinfo = mock(tableName, SelectedTab.class);
    EasyMock.expect(totalpayinfo.getName()).andReturn(tableName);
    List<ISelectedTab.ColMeta> cols = Lists.newArrayList();
    ISelectedTab.ColMeta cm = new ISelectedTab.ColMeta();
    cm.setName(colEntityId);
    cm.setType(new DataType(Types.VARCHAR, 6));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colNum);
    cm.setType(new DataType(Types.INTEGER));
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colId);
    cm.setType(new DataType(Types.VARCHAR, 32));
    cm.setPk(true);
    cols.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName(colCreateTime);
    cm.setType(new DataType(Types.BIGINT));
    cols.add(cm);
    EasyMock.expect(totalpayinfo.getCols()).andReturn(cols).anyTimes();
    selectedTabs.add(totalpayinfo);
    EasyMock.expect(dataxReader.getSelectedTabs()).andReturn(selectedTabs);
    EasyMock.expect(dataxProcessor.getReader(null)).andReturn(dataxReader);
    DataXClickhouseWriter dataXWriter = mock("dataXWriter", DataXClickhouseWriter.class);
    dataXWriter.initWriterTable(tableName, Collections.singletonList("jdbc:clickhouse://192.168.28.201:8123/tis"));
    ClickHouseDataSourceFactory sourceFactory = new ClickHouseDataSourceFactory();
    sourceFactory.userName = "default";
    sourceFactory.dbName = "tis";
    sourceFactory.password = "123456";
    sourceFactory.port = 8123;
    sourceFactory.nodeDesc = "192.168.28.201";
    EasyMock.expect(dataXWriter.getDataSourceFactory()).andReturn(sourceFactory);
    EasyMock.expect(dataxProcessor.getWriter(null)).andReturn(dataXWriter);
    Map<String, IDataxProcessor.TableAlias> aliasMap = new HashMap<>();
    IDataxProcessor.TableAlias tab = new IDataxProcessor.TableAlias(tableName);
    aliasMap.put(tableName, tab);
    EasyMock.expect(dataxProcessor.getTabAlias()).andReturn(aliasMap);
    this.replay();
    ClickHouseSinkFactory clickHouseSinkFactory = new ClickHouseSinkFactory();
    clickHouseSinkFactory.ignoringSendingException = true;
    clickHouseSinkFactory.maxBufferSize = 1;
    clickHouseSinkFactory.numRetries = 5;
    clickHouseSinkFactory.numWriters = 1;
    clickHouseSinkFactory.queueMaxCapacity = 1;
    clickHouseSinkFactory.timeout = 30;
    Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> sinkFuncs = clickHouseSinkFactory.createSinkFunction(dataxProcessor);
    assertTrue(sinkFuncs.size() > 0);
    StreamExecutionEnvironment env = StreamExecutionEnvironment.getExecutionEnvironment();
    env.setParallelism(1);
    DTO d = new DTO();
    d.setTableName(tableName);
    Map<String, Object> after = Maps.newHashMap();
    after.put(colEntityId, "334556");
    after.put(colNum, "5");
    after.put(colId, "123dsf124325253dsf123");
    after.put(colCreateTime, "20211113115959");
    d.setAfter(after);
    assertEquals(1, sinkFuncs.size());
    for (Map.Entry<IDataxProcessor.TableAlias, SinkFunction<DTO>> entry : sinkFuncs.entrySet()) {
        env.fromElements(new DTO[] { d }).addSink(entry.getValue()).name("clickhouse");
        break;
    }
    env.execute("testJob");
    Thread.sleep(5000);
    this.verifyAll();
}

Example 3 with ClickHouseDataSourceFactory

use of com.qlangtech.tis.plugin.ds.clickhouse.ClickHouseDataSourceFactory in project plugins by qlangtech.

the class TestDataXClickhouseWriter method createDataXWriter.

private static ClickHouseTest createDataXWriter() {
    String dbName = "tis";
    ClickHouseDataSourceFactory dsFactory = new ClickHouseDataSourceFactory();
    dsFactory.nodeDesc = "192.168.28.201";
    dsFactory.password = "123456";
    dsFactory.userName = "default";
    dsFactory.dbName = dbName;
    dsFactory.port = 8123;
    dsFactory.name = dbName;
    List<ISelectedTab.ColMeta> cmetas = Lists.newArrayList();
    IDataxProcessor.TableMap tableMap = new IDataxProcessor.TableMap(cmetas);
    tableMap.setFrom("application");
    tableMap.setTo(targetTableName);
    ISelectedTab.ColMeta cm = null;
    cm = new ISelectedTab.ColMeta();
    cm.setPk(true);
    cm.setName("customerregister_id");
    cm.setType(DataXReaderColType.STRING.dataType);
    cmetas.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName("waitingorder_id");
    cm.setType(DataXReaderColType.STRING.dataType);
    cmetas.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName("kind");
    cm.setType(DataXReaderColType.INT.dataType);
    cmetas.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName("create_time");
    cm.setType(DataXReaderColType.Long.dataType);
    cmetas.add(cm);
    cm = new ISelectedTab.ColMeta();
    cm.setName("last_ver");
    cm.setType(DataXReaderColType.INT.dataType);
    cmetas.add(cm);
    // tableMap.setSourceCols(cmetas);
    DataXClickhouseWriter writer = new DataXClickhouseWriter() {

        @Override
        public Class<?> getOwnerClass() {
            return DataXClickhouseWriter.class;
        }

        @Override
        public ClickHouseDataSourceFactory getDataSourceFactory() {
            // return super.getDataSourceFactory();
            return dsFactory;
        }
    };
    writer.template = DataXClickhouseWriter.getDftTemplate();
    writer.batchByteSize = 3456;
    writer.batchSize = 9527;
    writer.dbName = dbName;
    writer.writeMode = "insert";
    // writer.autoCreateTable = true;
    writer.postSql = "drop table @table";
    writer.preSql = "drop table @table";
    writer.dataXName = testDataXName;
    writer.dbName = dbName;
    return new ClickHouseTest(writer, tableMap);
}

Example 4 with ClickHouseDataSourceFactory

use of com.qlangtech.tis.plugin.ds.clickhouse.ClickHouseDataSourceFactory in project plugins by qlangtech.

the class DataXClickhouseWriter method getSubTask.

@Override
public IDataxContext getSubTask(Optional<IDataxProcessor.TableMap> tableMap) {
    if (!tableMap.isPresent()) {
        throw new IllegalArgumentException("tableMap shall be present");
    }
    IDataxProcessor.TableMap tmapper = tableMap.get();
    ClickHouseDataSourceFactory ds = this.getDataSourceFactory();
    ClickHouseWriterContext context = new ClickHouseWriterContext();
    context.setDataXName(this.dataXName);
    context.setBatchByteSize(this.batchByteSize);
    context.setBatchSize(this.batchSize);
    for (String jdbcUrl : ds.getJdbcUrls()) {
        context.setJdbcUrl(jdbcUrl);
        break;
    }
    if (StringUtils.isEmpty(context.getJdbcUrl())) {
        throw new IllegalStateException("jdbcUrl can not be null");
    }
    context.setUsername(ds.getUserName());
    context.setPassword(ds.password);
    context.setTable(tmapper.getTo());
    context.setWriteMode(this.writeMode);
    // };
    if (StringUtils.isNotEmpty(this.preSql)) {
        // context.setPreSql(helper.replacePlaceholders(this.preSql, resolver));
        context.setPreSql(this.preSql);
    }
    if (StringUtils.isNotEmpty(this.postSql)) {
        // context.setPostSql(helper.replacePlaceholders(this.postSql, resolver));
        context.setPostSql(this.postSql);
    }
    context.setCols(IDataxProcessor.TabCols.create(tableMap.get()));
    return context;
}