Examples with ColumnMetaData com.qlangtech.tis.plugin.ds.ColumnMetaData used on opensource projects

Example 11 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project tis by qlangtech.

the class ERRules method createDefaultErRule.

/**
 * 使用默认DumpNode创建ERRule并且持久化
 *
 * @param topology
 * @throws Exception
 */
public static void createDefaultErRule(SqlTaskNodeMeta.SqlDataFlowTopology topology) throws Exception {
    // 还没有定义erRule
    DependencyNode dumpNode = topology.getFirstDumpNode();
    DataSourceFactoryPluginStore dsStore = TIS.getDataBasePluginStore(new PostedDSProp(dumpNode.getDbName()));
    TISTable tab = dsStore.loadTableMeta(dumpNode.getName());
    // String topologyName, DependencyNode node, TargetColumnMeta targetColMetas
    Optional<ColumnMetaData> firstPK = tab.getReflectCols().stream().filter((col) -> col.isPk()).findFirst();
    if (!firstPK.isPresent()) {
        throw new IllegalStateException("table:" + dumpNode.parseEntityName() + " can not find relevant PK cols");
    }
    createErRule(topology.getName(), dumpNode, firstPK.get());
}

Example 12 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project tis by qlangtech.

the class TestSqlTaskNode method testDumpSqlReflect.

public void testDumpSqlReflect() {
    final String sql = "select A,b,c as cc,d,e as ee from A";
    Set<String> cols = Sets.newHashSet("a", "b", "c", "d", "e");
    List<ColumnMetaData> rows = SqlTaskNode.reflectTableCols(sql);
    for (ColumnMetaData r : rows) {
        assertTrue(r.getKey() + " must contain in set", cols.contains(r.getKey()));
    }
    assertEquals(5, rows.size());
}

Example 13 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class ClickHouseSinkFactory method createSinkFunction.

private SinkFunction<DTO> createSinkFunction(String dbName, final String targetTabName, ISelectedTab tab, String jdbcUrl, ClickHouseDataSourceFactory dsFactory) {
    // create props for sink
    Properties props = new Properties();
    props.put(ClickHouseSinkConst.TARGET_TABLE_NAME, targetTabName);
    Objects.requireNonNull(maxBufferSize, "maxBufferSize can not be null");
    props.put(ClickHouseSinkConst.MAX_BUFFER_SIZE, String.valueOf(maxBufferSize));
    Map<String, String> globalParams = Maps.newHashMap();
    // ClickHouse cluster properties
    // "http://" + clickHouse.getContainerIpAddress() + ":" + dockerActualPort
    List<String> clickhouseHosts = Lists.newArrayList();
    DBConfig dbCfg = dsFactory.getDbConfig();
    try {
        dbCfg.vistDbName((cfg, ip, _dbName) -> {
            clickhouseHosts.add("http://" + ip + ":" + dsFactory.port + "/?database=" + _dbName);
            return false;
        });
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
    if (clickhouseHosts.size() < 1) {
        throw new IllegalStateException("clickhouseHosts size can not small than 1");
    }
    globalParams.put(ClickHouseClusterSettings.CLICKHOUSE_HOSTS, clickhouseHosts.stream().collect(Collectors.joining(",")));
    globalParams.put(ClickHouseClusterSettings.CLICKHOUSE_USER, dsFactory.getUserName());
    globalParams.put(ClickHouseClusterSettings.CLICKHOUSE_PASSWORD, StringUtils.trimToEmpty(dsFactory.getPassword()));
    // sink common
    globalParams.put(ClickHouseSinkConst.TIMEOUT_SEC, String.valueOf(this.timeout));
    globalParams.put(ClickHouseSinkConst.FAILED_RECORDS_PATH, "/tmp/tis-clickhouse-sink");
    globalParams.put(ClickHouseSinkConst.NUM_WRITERS, String.valueOf(this.numWriters));
    globalParams.put(ClickHouseSinkConst.NUM_RETRIES, String.valueOf(this.numRetries));
    globalParams.put(ClickHouseSinkConst.QUEUE_MAX_CAPACITY, String.valueOf(this.queueMaxCapacity));
    globalParams.put(ClickHouseSinkConst.IGNORING_CLICKHOUSE_SENDING_EXCEPTION_ENABLED, String.valueOf(this.ignoringSendingException));
    List<ColumnMetaData> colsMeta = dsFactory.getTableMetadata(targetTabName);
    if (CollectionUtils.isEmpty(colsMeta)) {
        throw new IllegalStateException("targetTabName relevant colsMeta can not be empty");
    }
    return new TISClickHouseSink(globalParams, props, colsMeta.stream().map((c) -> new ColMeta(c.getKey(), c.getType())).collect(Collectors.toList()));
}

Example 14 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class SelectedTab method getContextTableCols.

public static List<Option> getContextTableCols(Function<List<ColumnMetaData>, Stream<ColumnMetaData>> func) {
    SuFormProperties.SuFormGetterContext context = SuFormProperties.subFormGetterProcessThreadLocal.get();
    if (context == null || context.plugin == null) {
        return Collections.emptyList();
    }
    Describable plugin = Objects.requireNonNull(context.plugin, "context.plugin can not be null");
    if (!(plugin instanceof DataSourceMeta)) {
        throw new IllegalStateException("plugin must be type of " + DataSourceMeta.class.getName() + ", now type of " + plugin.getClass().getName());
    }
    DataSourceMeta dsMeta = (DataSourceMeta) plugin;
    List<ColumnMetaData> cols = context.getContextAttr(KEY_TABLE_COLS, (key) -> dsMeta.getTableMetadata(context.getSubFormIdentityField()));
    return func.apply(cols).map((c) -> c).collect(Collectors.toList());
}

Example 15 with ColumnMetaData

use of com.qlangtech.tis.plugin.ds.ColumnMetaData in project plugins by qlangtech.

the class TisDataXTiDBReader method getTiKVDataSource.

private static final List<IDataSourceDumper> getTiKVDataSource(Configuration config, Optional<Long> regionId) {
    List<String> cols = config.getList("column", String.class);
    String tableName = config.getString("table");
    Configuration connection = config.getConfiguration("connection");
    TiKVDataSourceFactory sourceFactory = new TiKVDataSourceFactory();
    sourceFactory.pdAddrs = connection.getString("[0].pdAddrs");
    sourceFactory.dbName = connection.getString("[0].dbName");
    // sourceFactory.datetimeFormat = connection.getBool("datetimeFormat");
    if (StringUtils.isBlank(sourceFactory.pdAddrs) || StringUtils.isBlank(sourceFactory.dbName)) {
        throw new IllegalStateException("param 'pdAddrs' or 'dbName' can not be null,connection:" + connection.toJSON());
    }
    List<ColumnMetaData> tableMetadata = sourceFactory.getTableMetadata(tableName);
    TISTable table = new TISTable();
    table.setTableName(tableName);
    table.setReflectCols(tableMetadata.stream().filter((cmeta) -> cols.contains(cmeta.getKey())).collect(Collectors.toList()));
    DataDumpers dataDumpers = sourceFactory.getDataDumpers(table, regionId);
    List<IDataSourceDumper> dumpers = Lists.newArrayList(dataDumpers.dumpers);
    return dumpers;
}