Examples with BatchDataSource com.dtstack.taier.dao.domain.BatchDataSource used on opensource projects

Example 1 with BatchDataSource

use of com.dtstack.taier.dao.domain.BatchDataSource in project Taier by DTStack.

the class DatasourceService method getHivePartitions.

public Set<String> getHivePartitions(Long sourceId, String tableName) {
    BatchDataSource source = getOne(sourceId);
    JSONObject json = JSON.parseObject(source.getDataJson());
    Map<String, Object> kerberosConfig = this.fillKerberosConfig(sourceId);
    ISourceDTO sourceDTO = SourceDTOType.getSourceDTO(json, source.getType(), kerberosConfig, Maps.newHashMap());
    IClient iClient = ClientCache.getClient(source.getType());
    List<ColumnMetaDTO> partitionColumn = iClient.getPartitionColumn(sourceDTO, SqlQueryDTO.builder().tableName(tableName).build());
    Set<String> partitionNameSet = Sets.newHashSet();
    // 格式化分区信息 与hive保持一致
    if (CollectionUtils.isNotEmpty(partitionColumn)) {
        StringJoiner tempJoiner = new StringJoiner("=/", "", "=");
        for (ColumnMetaDTO column : partitionColumn) {
            tempJoiner.add(column.getKey());
        }
        partitionNameSet.add(tempJoiner.toString());
    }
    return partitionNameSet;
}

Example 2 with BatchDataSource

use of com.dtstack.taier.dao.domain.BatchDataSource in project Taier by DTStack.

the class DatasourceService method executeOnSpecifySourceWithOutResult.

/**
 * @param sourceId  数据源id
 * @param sql  拼写sql
 * @param targetSchema 只做doris入参,其他类型不用传
 */
private void executeOnSpecifySourceWithOutResult(Long sourceId, String sql, String targetSchema) {
    BatchDataSource source = getOne(sourceId);
    DataSourceType dataSourceType = DataSourceType.getSourceType(source.getType());
    if (!SUPPORT_CREATE_TABLE_DATASOURCES.contains(dataSourceType)) {
        throw new RdosDefineException(String.format("只支持创建%s数据源表", SUPPORT_CREATE_TABLE_DATASOURCES_NAMES));
    }
    JSONObject json = JSON.parseObject(source.getDataJson());
    try {
        Map<String, Object> kerberosConfig = fillKerberosConfig(sourceId);
        Map<String, Object> expandConfigPrepare = expandConfigPrepare(sourceId);
        ISourceDTO sourceDTO = SourceDTOType.getSourceDTO(json, source.getType(), kerberosConfig, expandConfigPrepare);
        IClient iClient = ClientCache.getClient(dataSourceType.getVal());
        Connection con = iClient.getCon(sourceDTO);
        DBUtil.executeSqlWithoutResultSet(con, sql, false);
    } catch (Exception e) {
        throw new RdosDefineException(e.getMessage() + "。 执行sql = " + sql, e);
    }
}

Example 3 with BatchDataSource

use of com.dtstack.taier.dao.domain.BatchDataSource in project Taier by DTStack.

the class DatasourceService method columnForSyncopate.

/**
 * 返回切分键需要的列名
 * <p>
 * 只支持关系型数据库 mysql\oracle\sqlserver\postgresql  的整型数据类型
 * 也不支持其他数据库。
 * 如果指定了不支持的类型，则忽略切分键功能，使用单通道进行同步。
 *
 * @param userId
 * @param sourceId
 * @param tableName
 * @return
 */
public Set<JSONObject> columnForSyncopate(Long userId, Long sourceId, String tableName, String schema) {
    BatchDataSource source = getOne(sourceId);
    if (Objects.isNull(RDBMSSourceType.getByDataSourceType(source.getType())) && !DataSourceType.INFLUXDB.getVal().equals(source.getType())) {
        LOGGER.error("切分键只支关系型数据库");
        throw new RdosDefineException("切分键只支持关系型数据库");
    }
    if (StringUtils.isEmpty(tableName)) {
        return new HashSet<>();
    }
    final StringBuffer newTableName = new StringBuffer();
    if (DataSourceType.SQLServer.getVal().equals(source.getType()) && StringUtils.isNotBlank(tableName)) {
        if (tableName.indexOf("[") == -1) {
            final String[] tableNames = tableName.split("\\.");
            for (final String name : tableNames) {
                newTableName.append("[").append(name).append("]").append(".");
            }
            tableName = newTableName.substring(0, newTableName.length() - 1);
        }
    }
    final List<JSONObject> tablecolumn = this.getTableColumn(source, tableName, schema);
    if (CollectionUtils.isNotEmpty(tablecolumn)) {
        List<String> numbers;
        if (DataSourceType.MySQL.getVal().equals(source.getType()) || DataSourceType.Polardb_For_MySQL.getVal().equals(source.getType()) || DataSourceType.TiDB.getVal().equals(source.getType())) {
            numbers = MYSQL_NUMBERS;
        } else if (DataSourceType.Oracle.getVal().equals(source.getType())) {
            numbers = ORACLE_NUMBERS;
        } else if (DataSourceType.SQLServer.getVal().equals(source.getType())) {
            numbers = SQLSERVER_NUMBERS;
        } else if (DataSourceType.PostgreSQL.getVal().equals(source.getType()) || DataSourceType.ADB_FOR_PG.getVal().equals(source.getType())) {
            numbers = POSTGRESQL_NUMBERS;
        } else if (DataSourceType.DB2.getVal().equals(source.getType())) {
            numbers = DB2_NUMBERS;
        } else if (DataSourceType.GBase_8a.getVal().equals(source.getType())) {
            numbers = GBASE_NUMBERS;
        } else if (DataSourceType.Clickhouse.getVal().equals(source.getType())) {
            numbers = CLICKHOUSE_NUMBERS;
        } else if (DataSourceType.DMDB.getVal().equals(source.getType())) {
            numbers = DMDB_NUMBERS;
        } else if (DataSourceType.GREENPLUM6.getVal().equals(source.getType())) {
            numbers = GREENPLUM_NUMBERS;
        } else if (DataSourceType.KINGBASE8.getVal().equals(source.getType())) {
            numbers = KINGBASE_NUMBERS;
        } else if (DataSourceType.INFLUXDB.getVal().equals(source.getType())) {
            numbers = INFLUXDB_NUMBERS;
        } else {
            throw new RdosDefineException("切分键只支持关系型数据库");
        }
        Map<JSONObject, String> twinsMap = new LinkedHashMap<>(tablecolumn.size() + 1);
        for (JSONObject twins : tablecolumn) {
            twinsMap.put(twins, twins.getString(TYPE));
        }
        Iterator<Map.Entry<JSONObject, String>> iterator = twinsMap.entrySet().iterator();
        while (iterator.hasNext()) {
            String type = getSimpleType(iterator.next().getValue());
            if (numbers.contains(type.toUpperCase())) {
                continue;
            }
            if (source.getType().equals(DataSourceType.Oracle.getVal())) {
                if ("number".equalsIgnoreCase(type)) {
                    continue;
                }
                Matcher numberMatcher1 = NUMBER_PATTERN.matcher(type);
                Matcher numberMatcher2 = NUMBER_PATTERN2.matcher(type);
                if (numberMatcher1.matches()) {
                    continue;
                } else if (numberMatcher2.matches()) {
                    int floatLength = Integer.parseInt(numberMatcher2.group(2));
                    if (floatLength <= 0) {
                        continue;
                    }
                }
            }
            iterator.remove();
        }
        // 为oracle加上默认切分键
        if (source.getType().equals(DataSourceType.Oracle.getVal())) {
            JSONObject keySet = new JSONObject();
            keySet.put("type", "NUMBER(38,0)");
            keySet.put("key", "ROW_NUMBER()");
            keySet.put("comment", "");
            twinsMap.put(keySet, "NUMBER(38,0)");
        }
        return twinsMap.keySet();
    }
    return Sets.newHashSet();
}

Example 4 with BatchDataSource

use of com.dtstack.taier.dao.domain.BatchDataSource in project Taier by DTStack.

the class DatasourceService method expandConfigPrepare.

/**
 * 拓展配置 ssl 配置信息等
 *
 * @param sourceId
 * @return
 */
public Map<String, Object> expandConfigPrepare(Long sourceId) {
    BatchDataSource sourceInfo = getOne(sourceId);
    JSONObject dataJson = JSONObject.parseObject(sourceInfo.getDataJson());
    String sftpDir = dataJson.getString(KEY_PATH);
    if (StringUtils.isEmpty(sftpDir)) {
        return Collections.emptyMap();
    }
    Map<String, Object> config = new HashMap<>();
    Map<String, String> sftpMap = getSftpMap(sourceInfo.getTenantId());
    DataSourceUtils.downloadFileFromSftp(sftpDir, DataSourceUtils.getLocalSslDir(sourceId), sftpMap, null);
    config.put(SSL_LOCAL_DIR, DataSourceUtils.getLocalSslDir(sourceId));
    return config;
}

Example 5 with BatchDataSource

use of com.dtstack.taier.dao.domain.BatchDataSource in project Taier by DTStack.

the class BatchTaskService method fillKerberosConfig.

/**
 * 下载检查kerberos配置
 *
 * @param sourceId
 * @return 返回该数据源的完整kerberos配置
 */
public Map<String, Object> fillKerberosConfig(Long sourceId) {
    BatchDataSource source = dataSourceService.getOne(sourceId);
    Long tenantId = tenantService.getDtTenantId(source.getTenantId());
    JSONObject dataJson = JSON.parseObject(source.getDataJson());
    JSONObject kerberosConfig = dataJson.getJSONObject(KERBEROS_CONFIG);
    if (MapUtils.isNotEmpty(kerberosConfig)) {
        String localKerberosConf = getLocalKerberosConf(sourceId);
        downloadKerberosFromSftp(kerberosConfig.getString(KERBEROS_DIR), localKerberosConf, tenantId, dataJson.getTimestamp(KERBEROS_FILE_TIMESTAMP));
        return handleKerberos(source.getType(), kerberosConfig, localKerberosConf);
    }
    return new HashMap<>();
}