Examples with ColumnMetaDTO com.dtstack.dtcenter.loader.dto.ColumnMetaDTO used on opensource projects

Example 1 with ColumnMetaDTO

use of com.dtstack.dtcenter.loader.dto.ColumnMetaDTO in project Taier by DTStack.

the class HiveBase method inferHdfsParams.

protected void inferHdfsParams() {
    if (inferred.compareAndSet(false, true) && StringUtils.isNotBlank(table)) {
        DataSourceType sourceType = DataSourceType.getSourceType(dataSourceType);
        JSONObject dataJson = new JSONObject();
        dataJson.put(SourceDTOType.JDBC_URL, jdbcUrl);
        dataJson.put(SourceDTOType.JDBC_USERNAME, username);
        dataJson.put(SourceDTOType.JDBC_PASSWORD, password);
        ISourceDTO sourceDTO = SourceDTOType.getSourceDTO(dataJson, sourceType.getVal(), kerberosConfig, Maps.newHashMap());
        IClient client = ClientCache.getClient(sourceType.getVal());
        Table tableInfo = client.getTable(sourceDTO, SqlQueryDTO.builder().tableName(this.table).build());
        List<ColumnMetaDTO> columnMetaData = tableInfo.getColumns();
        for (ColumnMetaDTO dto : columnMetaData) {
            if (!dto.getPart()) {
                fullColumnNames.add(dto.getKey());
                fullColumnTypes.add(dto.getType());
            } else {
                isPartitioned = true;
                partitionedBy.add(dto.getKey());
            }
        }
        if (isPartitioned) {
            ITable tableClient = ClientCache.getTable(sourceType.getVal());
            List<String> partitions = tableClient.showPartitions(sourceDTO, table);
            partitions.forEach(bean -> {
                partitionList.add(bean);
            });
        }
        this.dbName = tableInfo.getDb();
        this.path = tableInfo.getPath();
        this.fileType = tableInfo.getStoreType();
        this.fieldDelimiter = tableInfo.getDelim();
        this.isTransaction = tableInfo.getIsTransTable();
    }
    for (int i = 0; i < fullColumnNames.size(); i++) {
        for (Object col : column) {
            if (fullColumnNames.get(i).equals(((Map<String, Object>) col).get("key"))) {
                ((Map<String, Object>) col).put("index", i);
                break;
            }
        }
    }
}

Example 2 with ColumnMetaDTO

use of com.dtstack.dtcenter.loader.dto.ColumnMetaDTO in project Taier by DTStack.

the class JdbcServiceImpl method executeQueryWithVariables.

public List<List<Object>> executeQueryWithVariables(Long tenantId, Long userId, EScheduleJobType eScheduleJobType, String schema, String sql, List<String> variables, Integer limit, String taskParam) {
    List<List<Object>> returnList = new ArrayList<>();
    JdbcInfo jdbcInfo = Engine2DTOService.getJdbcInfo(tenantId, userId, eScheduleJobType);
    Integer maxRows = limit == null || limit == 0 ? jdbcInfo.getMaxRows() : limit;
    ISourceDTO iSourceDTO = Engine2DTOService.get(tenantId, userId, Engine2DTOService.jobTypeTransitionDataSourceType(eScheduleJobType, jdbcInfo.getVersion()).getVal(), schema, jdbcInfo);
    IClient client = ClientCache.getClient(iSourceDTO.getSourceType());
    // 率先获取Con，复用，为什么不使用try with resource，因为关闭捕获的异常太大了
    Connection con = client.getCon(iSourceDTO, taskParam);
    // 处理 variables SQL
    try {
        iSourceDTO.setConnection(con);
        List<Map<String, Object>> list;
        if (CollectionUtils.isNotEmpty(variables)) {
            variables.forEach(variable -> client.executeSqlWithoutResultSet(iSourceDTO, SqlQueryDTO.builder().sql(variable).limit(jdbcInfo.getMaxRows()).queryTimeout(jdbcInfo.getQueryTimeout()).build()));
            list = client.executeQuery(iSourceDTO, SqlQueryDTO.builder().sql(sql).limit(maxRows).queryTimeout(jdbcInfo.getQueryTimeout()).build());
        } else {
            list = client.executeQuery(iSourceDTO, SqlQueryDTO.builder().sql(sql).limit(maxRows).queryTimeout(jdbcInfo.getQueryTimeout()).build());
        }
        LOGGER.info("集群执行SQL查询，tenantId:{}，userId:{}，jobType:{}，schema:{}，sql:{}", tenantId, userId, eScheduleJobType.getType(), schema, sql);
        List<ColumnMetaDTO> columnMetaDataWithSql = client.getColumnMetaDataWithSql(iSourceDTO, SqlQueryDTO.builder().sql(sql).limit(0).queryTimeout(jdbcInfo.getQueryTimeout()).build());
        if (CollectionUtils.isNotEmpty(columnMetaDataWithSql)) {
            List<Object> column = new ArrayList<>();
            columnMetaDataWithSql.stream().forEach(bean -> {
                column.add(bean.getKey());
            });
            returnList.add(column);
        }
        // 数据源插件化 查询出值不符合要求  进行转化
        if (CollectionUtils.isNotEmpty(list)) {
            for (Map<String, Object> result : list) {
                List<Object> value = new ArrayList<>();
                result.values().forEach(bean -> {
                    value.add(bean);
                });
                returnList.add(value);
            }
        }
    } finally {
        iSourceDTO.setConnection(null);
        DBUtil.closeDBResources(null, null, con);
    }
    return returnList;
}

Example 3 with ColumnMetaDTO

use of com.dtstack.dtcenter.loader.dto.ColumnMetaDTO in project Taier by DTStack.

the class DatasourceService method getHivePartitions.

public Set<String> getHivePartitions(Long sourceId, String tableName) {
    BatchDataSource source = getOne(sourceId);
    JSONObject json = JSON.parseObject(source.getDataJson());
    Map<String, Object> kerberosConfig = this.fillKerberosConfig(sourceId);
    ISourceDTO sourceDTO = SourceDTOType.getSourceDTO(json, source.getType(), kerberosConfig, Maps.newHashMap());
    IClient iClient = ClientCache.getClient(source.getType());
    List<ColumnMetaDTO> partitionColumn = iClient.getPartitionColumn(sourceDTO, SqlQueryDTO.builder().tableName(tableName).build());
    Set<String> partitionNameSet = Sets.newHashSet();
    // 格式化分区信息 与hive保持一致
    if (CollectionUtils.isNotEmpty(partitionColumn)) {
        StringJoiner tempJoiner = new StringJoiner("=/", "", "=");
        for (ColumnMetaDTO column : partitionColumn) {
            tempJoiner.add(column.getKey());
        }
        partitionNameSet.add(tempJoiner.toString());
    }
    return partitionNameSet;
}

Example 4 with ColumnMetaDTO

use of com.dtstack.dtcenter.loader.dto.ColumnMetaDTO in project Taier by DTStack.

the class DatasourceService method getTableColumnIncludePart.

/**
 * 查询表所属字段 可以选择是否需要分区字段
 * @param source
 * @param tableName
 * @param part 是否需要分区字段
 * @return
 * @throws Exception
 */
private List<JSONObject> getTableColumnIncludePart(BatchDataSource source, String tableName, Boolean part, String schema) {
    try {
        if (source == null) {
            throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_DATA_SOURCE);
        }
        if (part == null) {
            part = false;
        }
        JSONObject dataJson = JSONObject.parseObject(source.getDataJson());
        Map<String, Object> kerberosConfig = fillKerberosConfig(source.getId());
        IClient iClient = ClientCache.getClient(source.getType());
        SqlQueryDTO sqlQueryDTO = SqlQueryDTO.builder().tableName(tableName).schema(schema).filterPartitionColumns(part).build();
        ISourceDTO iSourceDTO = SourceDTOType.getSourceDTO(dataJson, source.getType(), kerberosConfig, Maps.newHashMap());
        List<ColumnMetaDTO> columnMetaData = iClient.getColumnMetaData(iSourceDTO, sqlQueryDTO);
        List<JSONObject> list = new ArrayList<>();
        if (CollectionUtils.isNotEmpty(columnMetaData)) {
            for (ColumnMetaDTO columnMetaDTO : columnMetaData) {
                JSONObject jsonObject = JSON.parseObject(JSON.toJSONString(columnMetaDTO));
                jsonObject.put("isPart", columnMetaDTO.getPart());
                list.add(jsonObject);
            }
        }
        return list;
    } catch (DtCenterDefException e) {
        throw e;
    } catch (Exception e) {
        throw new RdosDefineException(ErrorCode.GET_COLUMN_ERROR, e);
    }
}

Example 5 with ColumnMetaDTO

use of com.dtstack.dtcenter.loader.dto.ColumnMetaDTO in project Taier by DTStack.

the class BatchTaskService method getTableColumnIncludePart.

/**
 * 查询表所属字段 可以选择是否需要分区字段
 * @param source
 * @param tableName
 * @param part 是否需要分区字段
 * @return
 * @throws Exception
 */
private List<JSONObject> getTableColumnIncludePart(BatchDataSource source, String tableName, Boolean part, String schema) {
    try {
        if (source == null) {
            throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_DATA_SOURCE);
        }
        if (part == null) {
            part = false;
        }
        JSONObject dataJson = JSONObject.parseObject(source.getDataJson());
        Map<String, Object> kerberosConfig = fillKerberosConfig(source.getId());
        IClient iClient = ClientCache.getClient(source.getType());
        SqlQueryDTO sqlQueryDTO = SqlQueryDTO.builder().tableName(tableName).schema(schema).filterPartitionColumns(part).build();
        ISourceDTO iSourceDTO = SourceDTOType.getSourceDTO(dataJson, source.getType(), kerberosConfig, Maps.newHashMap());
        List<ColumnMetaDTO> columnMetaData = iClient.getColumnMetaData(iSourceDTO, sqlQueryDTO);
        List<JSONObject> list = new ArrayList<>();
        if (CollectionUtils.isNotEmpty(columnMetaData)) {
            for (ColumnMetaDTO columnMetaDTO : columnMetaData) {
                JSONObject jsonObject = JSON.parseObject(JSON.toJSONString(columnMetaDTO));
                jsonObject.put("isPart", columnMetaDTO.getPart());
                list.add(jsonObject);
            }
        }
        return list;
    } catch (DtCenterDefException e) {
        throw e;
    } catch (Exception e) {
        throw new RdosDefineException(ErrorCode.GET_COLUMN_ERROR, e);
    }
}