Examples with BasicDorisStarRocksWriter com.qlangtech.tis.plugin.datax.BasicDorisStarRocksWriter used on opensource projects

Example 1 with BasicDorisStarRocksWriter

use of com.qlangtech.tis.plugin.datax.BasicDorisStarRocksWriter in project plugins by qlangtech.

the class StarRocksSinkFactory method createSinkFunction.

@Override
public Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> createSinkFunction(IDataxProcessor dataxProcessor) {
    Map<IDataxProcessor.TableAlias, SinkFunction<DTO>> sinkFuncs = Maps.newHashMap();
    IDataxProcessor.TableAlias tableName = null;
    // Map<String, IDataxProcessor.TableAlias> tabAlias = dataxProcessor.getTabAlias();
    BasicDorisStarRocksWriter dataXWriter = (BasicDorisStarRocksWriter) dataxProcessor.getWriter(null);
    Objects.requireNonNull(dataXWriter, "dataXWriter can not be null");
    BasicDorisStarRocksWriter.Separator separator = dataXWriter.getSeparator();
    IDataxReader reader = dataxProcessor.getReader(null);
    List<ISelectedTab> tabs = reader.getSelectedTabs();
    DorisSourceFactory dsFactory = dataXWriter.getDataSourceFactory();
    DBConfig dbConfig = dsFactory.getDbConfig();
    Map<String, IDataxProcessor.TableAlias> selectedTabs = dataxProcessor.getTabAlias();
    if (MapUtils.isEmpty(selectedTabs)) {
        throw new IllegalStateException("selectedTabs can not be empty");
    }
    for (Map.Entry<String, IDataxProcessor.TableAlias> tabAliasEntry : selectedTabs.entrySet()) {
        tableName = tabAliasEntry.getValue();
        Objects.requireNonNull(tableName, "tableName can not be null");
        if (StringUtils.isEmpty(tableName.getFrom())) {
            throw new IllegalStateException("tableName.getFrom() can not be empty");
        }
        AtomicReference<SinkFunction<DTO>> sinkFuncRef = new AtomicReference<>();
        final IDataxProcessor.TableAlias tabName = tableName;
        AtomicReference<Object[]> exceptionLoader = new AtomicReference<>();
        final String targetTabName = tableName.getTo();
        dbConfig.vistDbURL(false, (dbName, jdbcUrl) -> {
            try {
                Optional<ISelectedTab> selectedTab = tabs.stream().filter((tab) -> StringUtils.equals(tabName.getFrom(), tab.getName())).findFirst();
                if (!selectedTab.isPresent()) {
                    throw new IllegalStateException("target table:" + tabName.getFrom() + " can not find matched table in:[" + tabs.stream().map((t) -> t.getName()).collect(Collectors.joining(",")) + "]");
                }
                /**
                 * 需要先初始化表starrocks目标库中的表
                 */
                dataXWriter.initWriterTable(targetTabName, Collections.singletonList(jdbcUrl));
                sinkFuncRef.set(createSinkFunction(dbName, targetTabName, selectedTab.get(), jdbcUrl, dsFactory, separator));
            } catch (Throwable e) {
                exceptionLoader.set(new Object[] { jdbcUrl, e });
            }
        });
        if (exceptionLoader.get() != null) {
            Object[] error = exceptionLoader.get();
            throw new RuntimeException((String) error[0], (Throwable) error[1]);
        }
        Objects.requireNonNull(sinkFuncRef.get(), "sinkFunc can not be null");
        sinkFuncs.put(tableName, sinkFuncRef.get());
    }
    if (sinkFuncs.size() < 1) {
        throw new IllegalStateException("size of sinkFuncs can not be small than 1");
    }
    return sinkFuncs;
}