Examples with ISourceDTO com.dtstack.dtcenter.loader.dto.source.ISourceDTO used on opensource projects

Example 16 with ISourceDTO

use of com.dtstack.dtcenter.loader.dto.source.ISourceDTO in project Taier by DTStack.

the class JdbcServiceImpl method getAllDataBases.

@Override
public List<String> getAllDataBases(Long clusterId, EComponentType eComponentType, String schema) {
    ISourceDTO iSourceDTO = Engine2DTOService.getByClusterId(clusterId, eComponentType, schema);
    IClient client = ClientCache.getClient(iSourceDTO.getSourceType());
    LOGGER.info("集群查询底层获取所有数据库名称，clusterId:{}，eComponentType:{}，schema:{}", clusterId, eComponentType.getTypeCode(), schema);
    List<String> allDatabases = client.getAllDatabases(iSourceDTO, SqlQueryDTO.builder().build());
    return allDatabases;
}

Example 17 with ISourceDTO

use of com.dtstack.dtcenter.loader.dto.source.ISourceDTO in project Taier by DTStack.

the class JdbcServiceImpl method getTableList.

@Override
public List<String> getTableList(Long tenantId, EScheduleJobType eScheduleJobType, String schema) {
    ISourceDTO iSourceDTO = Engine2DTOService.get(tenantId, null, eScheduleJobType, schema);
    IClient client = ClientCache.getClient(iSourceDTO.getSourceType());
    List<String> tableList = client.getTableList(iSourceDTO, SqlQueryDTO.builder().build());
    LOGGER.info("集群查询底层获取所有表名称，tenantId:{}，jobType:{}，schema:{}", tenantId, eScheduleJobType.getType(), schema);
    return tableList;
}

Example 18 with ISourceDTO

use of com.dtstack.dtcenter.loader.dto.source.ISourceDTO in project Taier by DTStack.

the class JdbcServiceImpl method executeQueryWithVariables.

/**
 * 执行查询
 * @param tenantId
 * @param userId
 * @param eScheduleJobType
 * @param schema
 * @param sql
 * @param variables
 * @param connection
 * @return
 */
@Override
public List<List<Object>> executeQueryWithVariables(Long tenantId, Long userId, EScheduleJobType eScheduleJobType, String schema, String sql, List<String> variables, Connection connection) {
    List<List<Object>> returnList = new ArrayList<>();
    JdbcInfo jdbcInfo = Engine2DTOService.getJdbcInfo(tenantId, userId, eScheduleJobType);
    DataSourceType dataSourceType = Engine2DTOService.jobTypeTransitionDataSourceType(eScheduleJobType, jdbcInfo.getVersion());
    ISourceDTO iSourceDTO = Engine2DTOService.get(tenantId, userId, dataSourceType.getVal(), schema, jdbcInfo);
    IClient client = ClientCache.getClient(iSourceDTO.getSourceType());
    List<Map<String, Object>> list = null;
    iSourceDTO.setConnection(connection);
    // 处理 variables SQL
    if (CollectionUtils.isNotEmpty(variables)) {
        variables.forEach(variable -> client.executeSqlWithoutResultSet(iSourceDTO, SqlQueryDTO.builder().sql(variable).limit(jdbcInfo.getMaxRows()).queryTimeout(jdbcInfo.getQueryTimeout()).build()));
        list = client.executeQuery(iSourceDTO, SqlQueryDTO.builder().sql(sql).limit(jdbcInfo.getMaxRows()).queryTimeout(jdbcInfo.getQueryTimeout()).build());
    } else {
        list = client.executeQuery(iSourceDTO, SqlQueryDTO.builder().sql(sql).limit(jdbcInfo.getMaxRows()).queryTimeout(jdbcInfo.getQueryTimeout()).build());
    }
    LOGGER.info("集群执行SQL查询，tenantId:{}，userId:{}，jobType:{}，schema:{}，sql:{}", tenantId, userId, eScheduleJobType.getType(), schema, sql);
    // 数据源插件化 查询出值不符合要求  进行转化
    if (CollectionUtils.isNotEmpty(list)) {
        List<Object> column = new ArrayList<>();
        list.get(0).keySet().stream().forEach(bean -> {
            column.add(bean);
        });
        returnList.add(column);
        for (Map<String, Object> result : list) {
            List<Object> value = new ArrayList<>();
            result.values().forEach(bean -> {
                value.add(bean);
            });
            returnList.add(value);
        }
    }
    return returnList;
}

Example 19 with ISourceDTO

use of com.dtstack.dtcenter.loader.dto.source.ISourceDTO in project Taier by DTStack.

the class ImpalaSyncBuilder method tableLocation.

public JSONObject tableLocation(Map<String, Object> dataSource, String tableName, Map<String, Object> kerberos) {
    JSONObject result = new JSONObject();
    try {
        ISourceDTO iSourceDTO = SourceDTOType.getSourceDTO(new JSONObject(dataSource), DataSourceType.IMPALA.getVal(), kerberos, Maps.newHashMap());
        // 获取表存储文件类型
        String fileType = ImpalaUtils.getTableFileType(iSourceDTO, tableName);
        Assert.isTrue(StringUtils.isNotBlank(fileType), "暂不支持的hive表文件类型");
        if ("KUDU".equals(fileType)) {
            // 获取kudu配置信息
            Map<String, String> tableParams = ImpalaUtils.getImpalaKuduTableParams(iSourceDTO, tableName);
            String masterAddress = tableParams.get("kudu.master_addresses");
            String kuduTableName = tableParams.get("kudu.table_name");
            Assert.isTrue(StringUtils.isNotBlank(masterAddress) && StringUtils.isNotBlank(kuduTableName), "impala获取kudu表配置失败");
            result.put(TableLocationType.key(), TableLocationType.KUDU.getValue());
            result.put("masterAddresses", masterAddress);
            result.put("kuduTableName", kuduTableName);
            return result;
        }
        result.put(TableLocationType.key(), TableLocationType.HIVE.getValue());
        // 获取impala表信息 用于hdfs同步
        Map<String, Object> map = ImpalaUtils.getImpalaHiveTableDetailInfo(iSourceDTO, tableName);
        String path = (String) map.get("path");
        Assert.isTrue(StringUtils.isNotBlank(path), "无法获取impala hive表location路径");
        result.put("fileType", fileType);
        result.putAll(map);
        return result;
    } catch (Exception e) {
        LOGGER.error("tableLocation error ", e);
        if (e instanceof RdosDefineException) {
            throw (RdosDefineException) e;
        }
    }
    throw new RdosDefineException("获取impala表存储类型失败");
}

Example 20 with ISourceDTO

use of com.dtstack.dtcenter.loader.dto.source.ISourceDTO in project Taier by DTStack.

the class DatasourceService method tablelist.

/**
 * 数据同步-获得数据库中相关的表信息
 *
 * @param sourceId  数据源id
 * @param schema 查询的schema
 * @param name 模糊查询表名
 * @return
 */
public List<String> tablelist(Long sourceId, String schema, String name) {
    List<String> tables = new ArrayList<>();
    BatchDataSource source = getOne(sourceId);
    String dataJson = source.getDataJson();
    JSONObject json = JSON.parseObject(dataJson);
    // 查询的db
    String dataSource = schema;
    IClient client = ClientCache.getClient(source.getType());
    ISourceDTO sourceDTO = SourceDTOType.getSourceDTO(json, source.getType(), fillKerberosConfig(source.getId()), Maps.newHashMap());
    SqlQueryDTO sqlQueryDTO = SqlQueryDTO.builder().tableNamePattern(name).limit(5000).build();
    sqlQueryDTO.setView(true);
    sqlQueryDTO.setSchema(dataSource);
    // 如果是hive类型的数据源  过滤脏数据表 和 临时表
    tables = client.getTableList(sourceDTO, sqlQueryDTO);
    return tables;
}