Examples with JdbcInfo com.dtstack.taier.common.engine.JdbcInfo used on opensource projects

Example 1 with JdbcInfo

use of com.dtstack.taier.common.engine.JdbcInfo in project Taier by DTStack.

the class JdbcServiceImpl method executeQueryWithVariables.

public List<List<Object>> executeQueryWithVariables(Long tenantId, Long userId, EScheduleJobType eScheduleJobType, String schema, String sql, List<String> variables, Integer limit, String taskParam) {
    List<List<Object>> returnList = new ArrayList<>();
    JdbcInfo jdbcInfo = Engine2DTOService.getJdbcInfo(tenantId, userId, eScheduleJobType);
    Integer maxRows = limit == null || limit == 0 ? jdbcInfo.getMaxRows() : limit;
    ISourceDTO iSourceDTO = Engine2DTOService.get(tenantId, userId, Engine2DTOService.jobTypeTransitionDataSourceType(eScheduleJobType, jdbcInfo.getVersion()).getVal(), schema, jdbcInfo);
    IClient client = ClientCache.getClient(iSourceDTO.getSourceType());
    // 率先获取Con，复用，为什么不使用try with resource，因为关闭捕获的异常太大了
    Connection con = client.getCon(iSourceDTO, taskParam);
    // 处理 variables SQL
    try {
        iSourceDTO.setConnection(con);
        List<Map<String, Object>> list;
        if (CollectionUtils.isNotEmpty(variables)) {
            variables.forEach(variable -> client.executeSqlWithoutResultSet(iSourceDTO, SqlQueryDTO.builder().sql(variable).limit(jdbcInfo.getMaxRows()).queryTimeout(jdbcInfo.getQueryTimeout()).build()));
            list = client.executeQuery(iSourceDTO, SqlQueryDTO.builder().sql(sql).limit(maxRows).queryTimeout(jdbcInfo.getQueryTimeout()).build());
        } else {
            list = client.executeQuery(iSourceDTO, SqlQueryDTO.builder().sql(sql).limit(maxRows).queryTimeout(jdbcInfo.getQueryTimeout()).build());
        }
        LOGGER.info("集群执行SQL查询，tenantId:{}，userId:{}，jobType:{}，schema:{}，sql:{}", tenantId, userId, eScheduleJobType.getType(), schema, sql);
        List<ColumnMetaDTO> columnMetaDataWithSql = client.getColumnMetaDataWithSql(iSourceDTO, SqlQueryDTO.builder().sql(sql).limit(0).queryTimeout(jdbcInfo.getQueryTimeout()).build());
        if (CollectionUtils.isNotEmpty(columnMetaDataWithSql)) {
            List<Object> column = new ArrayList<>();
            columnMetaDataWithSql.stream().forEach(bean -> {
                column.add(bean.getKey());
            });
            returnList.add(column);
        }
        // 数据源插件化 查询出值不符合要求  进行转化
        if (CollectionUtils.isNotEmpty(list)) {
            for (Map<String, Object> result : list) {
                List<Object> value = new ArrayList<>();
                result.values().forEach(bean -> {
                    value.add(bean);
                });
                returnList.add(value);
            }
        }
    } finally {
        iSourceDTO.setConnection(null);
        DBUtil.closeDBResources(null, null, con);
    }
    return returnList;
}

Example 2 with JdbcInfo

use of com.dtstack.taier.common.engine.JdbcInfo in project Taier by DTStack.

the class HadoopDataDownloadService method queryDataFromTempTable.

/**
 * 从临时表获取表数据
 *
 * @param tenantId
 * @param tableName
 * @param db
 * @return
 * @throws Exception
 */
public List<Object> queryDataFromTempTable(Long tenantId, String tableName, String db) throws Exception {
    DataSourceType dataSourceType = datasourceService.getHadoopDefaultDataSourceByTenantId(tenantId);
    IDownload downloader = getDownloader(tenantId, tableName, db, dataSourceType.getVal());
    List<Object> result = new ArrayList<>();
    List<String> alias = downloader.getMetaInfo();
    result.add(alias);
    JdbcInfo jdbcInfo = Engine2DTOService.getJdbcInfo(tenantId, null, DataSourceTypeJobTypeMapping.getTaskTypeByDataSourceType(dataSourceType.getVal()));
    int readCounter = 0;
    while (!downloader.reachedEnd() && readCounter < jdbcInfo.getMaxRows()) {
        List<String> row = (List<String>) downloader.readNext();
        result.add(row);
        readCounter++;
    }
    return result;
}

Example 3 with JdbcInfo

use of com.dtstack.taier.common.engine.JdbcInfo in project Taier by DTStack.

the class HadoopDataDownloadService method getDownloader.

public IDownload getDownloader(Long tenantId, String tableName, String db, List<String> columns, String partition, Integer dataSourceType) {
    Map<String, Object> hadoop = Engine2DTOService.getHdfs(tenantId);
    JdbcInfo jdbcInfo = Engine2DTOService.getJdbcInfo(tenantId, null, DataSourceTypeJobTypeMapping.getTaskTypeByDataSourceType(dataSourceType));
    try {
        return new HiveSelectDownload(hadoop, jdbcInfo, null, null, true, tenantId, db, tableName, partition, dataSourceType);
    } catch (Exception e) {
        LOGGER.error("", e);
        return null;
    }
}

Example 4 with JdbcInfo

use of com.dtstack.taier.common.engine.JdbcInfo in project Taier by DTStack.

the class JdbcServiceImpl method executeQueryMapResult.

@Override
public List<Map<String, Object>> executeQueryMapResult(Long tenantId, Long userId, EScheduleJobType eScheduleJobType, String schema, String sql) {
    JdbcInfo jdbcInfo = Engine2DTOService.getJdbcInfo(tenantId, userId, eScheduleJobType);
    ISourceDTO iSourceDTO = Engine2DTOService.get(tenantId, userId, Engine2DTOService.jobTypeTransitionDataSourceType(eScheduleJobType, jdbcInfo.getVersion()).getVal(), schema, jdbcInfo);
    IClient client = ClientCache.getClient(iSourceDTO.getSourceType());
    LOGGER.info("集群执行SQL查询，tenantId:{}，userId:{}，jobType:{}，schema:{}，sql:{}", tenantId, userId, eScheduleJobType.getType(), schema, sql);
    List<Map<String, Object>> list = client.executeQuery(iSourceDTO, SqlQueryDTO.builder().sql(sql).limit(jdbcInfo.getMaxRows()).queryTimeout(jdbcInfo.getQueryTimeout()).build());
    return list;
}

Example 5 with JdbcInfo

use of com.dtstack.taier.common.engine.JdbcInfo in project Taier by DTStack.

the class JdbcServiceImpl method executeQueryWithVariables.

/**
 * 执行查询
 * @param tenantId
 * @param userId
 * @param eScheduleJobType
 * @param schema
 * @param sql
 * @param variables
 * @param connection
 * @return
 */
@Override
public List<List<Object>> executeQueryWithVariables(Long tenantId, Long userId, EScheduleJobType eScheduleJobType, String schema, String sql, List<String> variables, Connection connection) {
    List<List<Object>> returnList = new ArrayList<>();
    JdbcInfo jdbcInfo = Engine2DTOService.getJdbcInfo(tenantId, userId, eScheduleJobType);
    DataSourceType dataSourceType = Engine2DTOService.jobTypeTransitionDataSourceType(eScheduleJobType, jdbcInfo.getVersion());
    ISourceDTO iSourceDTO = Engine2DTOService.get(tenantId, userId, dataSourceType.getVal(), schema, jdbcInfo);
    IClient client = ClientCache.getClient(iSourceDTO.getSourceType());
    List<Map<String, Object>> list = null;
    iSourceDTO.setConnection(connection);
    // 处理 variables SQL
    if (CollectionUtils.isNotEmpty(variables)) {
        variables.forEach(variable -> client.executeSqlWithoutResultSet(iSourceDTO, SqlQueryDTO.builder().sql(variable).limit(jdbcInfo.getMaxRows()).queryTimeout(jdbcInfo.getQueryTimeout()).build()));
        list = client.executeQuery(iSourceDTO, SqlQueryDTO.builder().sql(sql).limit(jdbcInfo.getMaxRows()).queryTimeout(jdbcInfo.getQueryTimeout()).build());
    } else {
        list = client.executeQuery(iSourceDTO, SqlQueryDTO.builder().sql(sql).limit(jdbcInfo.getMaxRows()).queryTimeout(jdbcInfo.getQueryTimeout()).build());
    }
    LOGGER.info("集群执行SQL查询，tenantId:{}，userId:{}，jobType:{}，schema:{}，sql:{}", tenantId, userId, eScheduleJobType.getType(), schema, sql);
    // 数据源插件化 查询出值不符合要求  进行转化
    if (CollectionUtils.isNotEmpty(list)) {
        List<Object> column = new ArrayList<>();
        list.get(0).keySet().stream().forEach(bean -> {
            column.add(bean);
        });
        returnList.add(column);
        for (Map<String, Object> result : list) {
            List<Object> value = new ArrayList<>();
            result.values().forEach(bean -> {
                value.add(bean);
            });
            returnList.add(value);
        }
    }
    return returnList;
}