Examples with ISourceDTO com.dtstack.dtcenter.loader.dto.source.ISourceDTO used on opensource projects

Example 6 with ISourceDTO

use of com.dtstack.dtcenter.loader.dto.source.ISourceDTO in project Taier by DTStack.

the class DatasourceService method getTableColumnIncludePart.

/**
 * 查询表所属字段 可以选择是否需要分区字段
 * @param source
 * @param tableName
 * @param part 是否需要分区字段
 * @return
 * @throws Exception
 */
private List<JSONObject> getTableColumnIncludePart(BatchDataSource source, String tableName, Boolean part, String schema) {
    try {
        if (source == null) {
            throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_DATA_SOURCE);
        }
        if (part == null) {
            part = false;
        }
        JSONObject dataJson = JSONObject.parseObject(source.getDataJson());
        Map<String, Object> kerberosConfig = fillKerberosConfig(source.getId());
        IClient iClient = ClientCache.getClient(source.getType());
        SqlQueryDTO sqlQueryDTO = SqlQueryDTO.builder().tableName(tableName).schema(schema).filterPartitionColumns(part).build();
        ISourceDTO iSourceDTO = SourceDTOType.getSourceDTO(dataJson, source.getType(), kerberosConfig, Maps.newHashMap());
        List<ColumnMetaDTO> columnMetaData = iClient.getColumnMetaData(iSourceDTO, sqlQueryDTO);
        List<JSONObject> list = new ArrayList<>();
        if (CollectionUtils.isNotEmpty(columnMetaData)) {
            for (ColumnMetaDTO columnMetaDTO : columnMetaData) {
                JSONObject jsonObject = JSON.parseObject(JSON.toJSONString(columnMetaDTO));
                jsonObject.put("isPart", columnMetaDTO.getPart());
                list.add(jsonObject);
            }
        }
        return list;
    } catch (DtCenterDefException e) {
        throw e;
    } catch (Exception e) {
        throw new RdosDefineException(ErrorCode.GET_COLUMN_ERROR, e);
    }
}

Example 7 with ISourceDTO

use of com.dtstack.dtcenter.loader.dto.source.ISourceDTO in project Taier by DTStack.

the class BatchTaskService method getTableColumnIncludePart.

/**
 * 查询表所属字段 可以选择是否需要分区字段
 * @param source
 * @param tableName
 * @param part 是否需要分区字段
 * @return
 * @throws Exception
 */
private List<JSONObject> getTableColumnIncludePart(BatchDataSource source, String tableName, Boolean part, String schema) {
    try {
        if (source == null) {
            throw new RdosDefineException(ErrorCode.CAN_NOT_FIND_DATA_SOURCE);
        }
        if (part == null) {
            part = false;
        }
        JSONObject dataJson = JSONObject.parseObject(source.getDataJson());
        Map<String, Object> kerberosConfig = fillKerberosConfig(source.getId());
        IClient iClient = ClientCache.getClient(source.getType());
        SqlQueryDTO sqlQueryDTO = SqlQueryDTO.builder().tableName(tableName).schema(schema).filterPartitionColumns(part).build();
        ISourceDTO iSourceDTO = SourceDTOType.getSourceDTO(dataJson, source.getType(), kerberosConfig, Maps.newHashMap());
        List<ColumnMetaDTO> columnMetaData = iClient.getColumnMetaData(iSourceDTO, sqlQueryDTO);
        List<JSONObject> list = new ArrayList<>();
        if (CollectionUtils.isNotEmpty(columnMetaData)) {
            for (ColumnMetaDTO columnMetaDTO : columnMetaData) {
                JSONObject jsonObject = JSON.parseObject(JSON.toJSONString(columnMetaDTO));
                jsonObject.put("isPart", columnMetaDTO.getPart());
                list.add(jsonObject);
            }
        }
        return list;
    } catch (DtCenterDefException e) {
        throw e;
    } catch (Exception e) {
        throw new RdosDefineException(ErrorCode.GET_COLUMN_ERROR, e);
    }
}

Example 8 with ISourceDTO

use of com.dtstack.dtcenter.loader.dto.source.ISourceDTO in project Taier by DTStack.

the class DatasourceService method preview.

/**
 * 数据同步-获得预览数据，默认展示3条
 *
 * @param sourceId  数据源id
 * @param tableName 表名
 * @return
 * @author toutian
 */
public JSONObject preview(Long sourceId, String tableName, String schema) {
    BatchDataSource source = getOne(sourceId);
    StringBuffer newTableName = new StringBuffer();
    if (DataSourceType.SQLServer.getVal().equals(source.getType()) && StringUtils.isNotBlank(tableName)) {
        if (tableName.indexOf("[") == -1) {
            final String[] tableNames = tableName.split("\\.");
            for (final String name : tableNames) {
                newTableName.append("[").append(name).append("]").append(".");
            }
            tableName = newTableName.substring(0, newTableName.length() - 1);
        }
    }
    String dataJson = source.getDataJson();
    JSONObject json = JSON.parseObject(dataJson);
    // 获取字段信息
    List<String> columnList = new ArrayList<String>();
    // 获取数据
    List<List<String>> dataList = new ArrayList<List<String>>();
    try {
        Map<String, Object> kerberosConfig = fillKerberosConfig(source.getId());
        List<JSONObject> columnJson = getTableColumn(source, tableName, schema);
        if (CollectionUtils.isNotEmpty(columnJson)) {
            for (JSONObject columnMetaDTO : columnJson) {
                columnList.add(columnMetaDTO.getString("key"));
            }
        }
        IClient iClient = ClientCache.getClient(source.getType());
        ISourceDTO iSourceDTO = SourceDTOType.getSourceDTO(json, source.getType(), kerberosConfig, Maps.newHashMap());
        SqlQueryDTO sqlQueryDTO = SqlQueryDTO.builder().schema(schema).tableName(tableName).previewNum(3).build();
        dataList = iClient.getPreview(iSourceDTO, sqlQueryDTO);
        if (DataSourceType.getRDBMS().contains(source.getType())) {
            // 因为会把字段名也会返回 所以要去除第一行
            dataList = dataList.subList(1, dataList.size());
        }
    } catch (Exception e) {
        LOGGER.error("datasource preview end with error.", e);
        throw new RdosDefineException(String.format("%s获取预览数据失败", source.getDataName()), e);
    }
    JSONObject preview = new JSONObject(2);
    preview.put("columnList", columnList);
    preview.put("dataList", dataList);
    return preview;
}

Example 9 with ISourceDTO

use of com.dtstack.dtcenter.loader.dto.source.ISourceDTO in project Taier by DTStack.

the class DatasourceService method checkConnectionWithConf.

/**
 * 数据源连通性测试
 * @param source
 * @param confMap
 * @param localKerberosPath
 * @return
 */
public Boolean checkConnectionWithConf(DataSourceVO source, Map<String, Object> confMap, String localKerberosPath) {
    DataSourceTypeEnum typeEnum = DataSourceTypeEnum.typeVersionOf(source.getDataType(), source.getDataVersion());
    if (MapUtils.isEmpty(confMap) && source.getId() > 0L) {
        confMap = fillKerberosConfig(source.getId());
        localKerberosPath = kerberosService.getLocalKerberosPath(source.getId());
    }
    if (DataSourceTypeEnum.ADB_PostgreSQL == typeEnum) {
        typeEnum = DataSourceTypeEnum.PostgreSQL;
    }
    // 替换相对绝对路径
    Map<String, Object> tempConfMap = null;
    if (MapUtils.isNotEmpty(confMap)) {
        tempConfMap = Maps.newHashMap(confMap);
        IKerberos kerberos = ClientCache.getKerberos(typeEnum.getVal());
        kerberos.prepareKerberosForConnect(tempConfMap, localKerberosPath);
    }
    // 测试连通性
    ISourceDTO sourceDTO = SourceDTOType.getSourceDTO(source.getDataJson(), typeEnum.getVal(), tempConfMap, Maps.newHashMap());
    return ClientCache.getClient(typeEnum.getVal()).testCon(sourceDTO);
}

Example 10 with ISourceDTO

use of com.dtstack.dtcenter.loader.dto.source.ISourceDTO in project Taier by DTStack.

the class DatasourceService method getTableInfo.

/**
 * 获取表信息
 *
 * @param sourceType
 * @param table
 * @param dataJson
 * @param kerberosConfig
 * @return
 */
private com.dtstack.dtcenter.loader.dto.Table getTableInfo(Integer sourceType, String table, JSONObject dataJson, Map<String, Object> kerberosConfig) {
    IClient client = ClientCache.getClient(sourceType);
    ISourceDTO sourceDTO = SourceDTOType.getSourceDTO(dataJson, sourceType, kerberosConfig, Maps.newHashMap());
    com.dtstack.dtcenter.loader.dto.Table tableInfo = client.getTable(sourceDTO, SqlQueryDTO.builder().tableName(table).build());
    return tableInfo;
}