Examples with DataSourceColumnRequest com.webank.wedatasphere.qualitis.rule.request.DataSourceColumnRequest used on opensource projects

Example 1 with DataSourceColumnRequest

use of com.webank.wedatasphere.qualitis.rule.request.DataSourceColumnRequest in project Qualitis by WeBankFinTech.

the class AddFileRuleRequestBuilder method solveDatasource.

private void solveDatasource(String datasource) throws UnExpectedRequestException {
    List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>();
    DataSourceRequest dataSourceRequest = new DataSourceRequest();
    String clusterName;
    String database;
    String table;
    String filter;
    String[] datasourceStrs = datasource.split(SpecCharEnum.COLON.getValue());
    if (datasourceStrs.length > TWO) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    String[] datasources = datasourceStrs[0].split(SpecCharEnum.PERIOD.getValue());
    filter = datasourceStrs.length >= TWO ? datasourceStrs[1] : "";
    if (datasources.length != FOUR && datasources.length != THREE) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    clusterName = datasources[0];
    database = datasources[1];
    table = datasources[2];
    dataSourceRequest.setClusterName(clusterName);
    dataSourceRequest.setDbName(database);
    dataSourceRequest.setTableName(table);
    dataSourceRequest.setColNames(dataSourceColumnRequests);
    dataSourceRequest.setFilter(filter);
    dataSourceRequest.setProxyUser(userName);
    this.addFileRuleRequest.setDatasource(dataSourceRequest);
}

Example 2 with DataSourceColumnRequest

use of com.webank.wedatasphere.qualitis.rule.request.DataSourceColumnRequest in project Qualitis by WeBankFinTech.

the class AddRuleRequestBuilder method solveDatasource.

private void solveDatasource(String datasource) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    List<DataSourceRequest> dataSourceRequests = new ArrayList<>(1);
    List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>(1);
    DataSourceRequest dataSourceRequest = new DataSourceRequest();
    String clusterName;
    String database;
    String table;
    String col;
    String filter;
    String[] datasourceStrs = datasource.split(SpecCharEnum.COLON.getValue());
    if (datasourceStrs.length > TWO) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    String dbAndTable = datasourceStrs[0];
    String dataSourceId = "";
    Matcher matcherId = DATA_SOURCE_ID.matcher(dbAndTable.toUpperCase());
    String dataSourceName = "";
    Matcher matcherName = DATA_SOURCE_NAME.matcher(dbAndTable.toUpperCase());
    while (matcherId.find()) {
        String group = matcherId.group();
        dataSourceId = group.replace(".(", "").replace(")", "").split("=")[1];
        int startIndex = dbAndTable.toUpperCase().indexOf(group);
        String replaceStr = dbAndTable.substring(startIndex, startIndex + group.length());
        dbAndTable = dbAndTable.replace(replaceStr, "");
    }
    if (StringUtils.isBlank(dataSourceId)) {
        while (matcherName.find()) {
            String group = matcherName.group();
            int startIndex = dbAndTable.toUpperCase().indexOf(group);
            String replaceStr = dbAndTable.substring(startIndex, startIndex + group.length());
            dataSourceName = replaceStr.replace(".(", "").replace(")", "").split("=")[1];
            dbAndTable = dbAndTable.replace(replaceStr, "");
        }
    }
    String[] datasources = dbAndTable.split(SpecCharEnum.PERIOD.getValue());
    filter = datasourceStrs[1];
    if (datasources.length != FOUR && datasources.length != THREE) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    clusterName = datasources[0];
    database = datasources[1];
    table = datasources[2];
    col = datasources.length >= FOUR ? datasources[3] : "";
    List<ColumnInfoDetail> cols = metaDataClient.getColumnInfo(clusterName, database, table, userName);
    if (StringUtils.isNotBlank(dataSourceId)) {
        LOGGER.info("Find data source connect. Data source ID: " + dataSourceId);
        dataSourceRequest.setLinkisDataSourceId(Long.parseLong(dataSourceId));
        GeneralResponse<Map> response = metaDataClient.getDataSourceInfoDetail(clusterName, userName, dataSourceRequest.getLinkisDataSourceId(), null);
        cols = metaDataClient.getColumnsByDataSource(clusterName, userName, dataSourceRequest.getLinkisDataSourceId(), database, table).getContent();
        Map dataSourceInfo = ((Map) response.getData().get("info"));
        String dataSourceInfoName = (String) dataSourceInfo.get("dataSourceName");
        String dataSourceInfoType = (String) ((Map) dataSourceInfo.get("dataSourceType")).get("name");
        dataSourceRequest.setLinkisDataSourceName(dataSourceInfoName);
        dataSourceRequest.setLinkisDataSourceType(dataSourceInfoType);
    } else if (StringUtils.isNotBlank(dataSourceName)) {
        LOGGER.info("Find data source connect. Data source name: " + dataSourceName);
        GeneralResponse<Map> response = metaDataClient.getDataSourceInfoDetailByName(clusterName, userName, dataSourceName);
        Map dataSourceInfo = ((Map) response.getData().get("info"));
        String dataSourceInfoName = (String) dataSourceInfo.get("dataSourceName");
        String dataSourceInfoType = (String) ((Map) dataSourceInfo.get("dataSourceType")).get("name");
        Integer currentDataSourceId = (Integer) dataSourceInfo.get("id");
        dataSourceRequest.setLinkisDataSourceId(currentDataSourceId.longValue());
        cols = metaDataClient.getColumnsByDataSource(clusterName, userName, currentDataSourceId.longValue(), database, table).getContent();
        dataSourceRequest.setLinkisDataSourceName(dataSourceInfoName);
        dataSourceRequest.setLinkisDataSourceType(dataSourceInfoType);
    }
    // For one or more fields
    if (StringUtils.isBlank(col)) {
        LOGGER.info("Table count check.");
    } else {
        boolean blackList = false;
        List<String> colsInDatasource = new ArrayList<>();
        if (col.startsWith(SpecCharEnum.MINUS.getValue())) {
            blackList = true;
            col = col.replace(SpecCharEnum.MINUS.getValue(), "");
        }
        if (col.contains(SpecCharEnum.LEFT_BRACKET.getValue()) && col.contains(SpecCharEnum.RIGHT_BRACKET.getValue())) {
            col = col.replace(SpecCharEnum.LEFT_BRACKET.getValue(), "").replace(SpecCharEnum.RIGHT_BRACKET.getValue(), "");
            for (String currentCol : col.split(SpecCharEnum.COMMA.getValue())) {
                colsInDatasource.add(currentCol);
            }
        } else {
            colsInDatasource.add(col);
        }
        if (blackList) {
            for (ColumnInfoDetail columnInfoDetail : cols) {
                if (colsInDatasource.contains(columnInfoDetail.getFieldName())) {
                    continue;
                } else {
                    DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest(columnInfoDetail.getFieldName(), columnInfoDetail.getDataType());
                    dataSourceColumnRequests.add(dataSourceColumnRequest);
                }
            }
        } else {
            for (String colName : colsInDatasource) {
                String type = cols.stream().filter(field -> field.getFieldName().equals(colName)).map(ColumnInfoDetail::getDataType).collect(Collectors.joining());
                DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest(colName, type);
                dataSourceColumnRequests.add(dataSourceColumnRequest);
            }
        }
        dataSourceRequest.setBlackList(blackList);
    }
    dataSourceRequest.setClusterName(clusterName);
    dataSourceRequest.setDbName(database);
    dataSourceRequest.setTableName(table);
    dataSourceRequest.setColNames(dataSourceColumnRequests);
    dataSourceRequest.setFilter(filter);
    dataSourceRequest.setProxyUser(userName);
    dataSourceRequests.add(dataSourceRequest);
    this.addRuleRequest.setDatasource(dataSourceRequests);
}

Example 3 with DataSourceColumnRequest

use of com.webank.wedatasphere.qualitis.rule.request.DataSourceColumnRequest in project Qualitis by WeBankFinTech.

the class RuleBatchServiceImpl method getDataSourceRequest.

private void getDataSourceRequest(List<DataSourceRequest> dataSourceRequests, ExcelTemplateRuleByProject excelTemplateRule) {
    String clusterName = excelTemplateRule.getCluster();
    if (!StringUtils.isBlank(clusterName)) {
        DataSourceRequest dataSourceRequest = new DataSourceRequest();
        if (excelTemplateRule.getLinkisDataSourceId() != null) {
            dataSourceRequest.setLinkisDataSourceId(Long.parseLong(excelTemplateRule.getLinkisDataSourceId()));
            dataSourceRequest.setLinkisDataSourceName(excelTemplateRule.getLinkisDataSourceName());
            dataSourceRequest.setLinkisDataSourceType(excelTemplateRule.getLinkisDataSourceType());
        }
        dataSourceRequest.setClusterName(excelTemplateRule.getCluster());
        dataSourceRequest.setProxyUser(excelTemplateRule.getProxyUser());
        dataSourceRequest.setTableName(excelTemplateRule.getTableName());
        dataSourceRequest.setDbName(excelTemplateRule.getDbName());
        dataSourceRequest.setFilter(excelTemplateRule.getFilter());
        String colNamesOrigin = excelTemplateRule.getColumnNames();
        List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>();
        if (!StringUtils.isBlank(colNamesOrigin)) {
            String[] colNamesSplit = colNamesOrigin.split(SpecCharEnum.VERTICAL_BAR.getValue());
            for (String str : colNamesSplit) {
                DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest();
                dataSourceColumnRequest.setColumnName(str.split(":")[0]);
                dataSourceColumnRequest.setDataType(str.split(":")[1]);
                dataSourceColumnRequests.add(dataSourceColumnRequest);
            }
        }
        // Default all column will be white list field to be imported.
        dataSourceRequest.setBlackList(false);
        dataSourceRequest.setColNames(dataSourceColumnRequests);
        dataSourceRequests.add(dataSourceRequest);
    }
}

Example 4 with DataSourceColumnRequest

use of com.webank.wedatasphere.qualitis.rule.request.DataSourceColumnRequest in project Qualitis by WeBankFinTech.

the class RuleDataSourceServiceImpl method saveCustomColumn.

private List<DataSourceColumnRequest> saveCustomColumn(String clusterName, String db, String table, String loginUser, String csId, String nodeName, String funcContent, boolean fps, boolean cs) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>();
    List<ColumnInfoDetail> cols = new ArrayList<>();
    if (cs) {
        GetUserTableByCsIdRequest getUserTableByCsIdRequest = new GetUserTableByCsIdRequest();
        getUserTableByCsIdRequest.setClusterName(clusterName);
        getUserTableByCsIdRequest.setLoginUser(loginUser);
        getUserTableByCsIdRequest.setCsId(csId);
        getUserTableByCsIdRequest.setNodeName(nodeName);
        DataInfo<CsTableInfoDetail> csTableInfoDetails = metaDataClient.getTableByCsId(getUserTableByCsIdRequest);
        if (csTableInfoDetails.getTotalCount() == 0 && !fps) {
            LOGGER.info("Cannot find context service table with existed rules!");
            throw new UnExpectedRequestException("Table in sql {&DOES_NOT_EXIST}");
        }
        for (CsTableInfoDetail csTableInfoDetail : csTableInfoDetails.getContent()) {
            if (csTableInfoDetail.getTableName().equals(table)) {
                GetUserColumnByCsRequest getUserColumnByCsRequest = new GetUserColumnByCsRequest();
                getUserColumnByCsRequest.setClusterName(clusterName);
                getUserColumnByCsRequest.setContextKey(csTableInfoDetail.getContextKey());
                getUserColumnByCsRequest.setCsId(csId);
                getUserColumnByCsRequest.setLoginUser(loginUser);
                cols = metaDataClient.getColumnByCsId(getUserColumnByCsRequest).getContent();
                if (CollectionUtils.isEmpty(cols) && !fps) {
                    throw new UnExpectedRequestException("Table in sql {&DOES_NOT_EXIST}");
                }
            }
        }
    } else {
        cols = metaDataClient.getColumnInfo(clusterName, db, table, loginUser);
        if (CollectionUtils.isEmpty(cols) && !fps) {
            throw new UnExpectedRequestException("Table in sql {&DOES_NOT_EXIST}");
        }
    }
    // Save column info in use.
    for (ColumnInfoDetail detail : cols) {
        if (funcContent != null && funcContent.contains(detail.getFieldName())) {
            dataSourceColumnRequests.add(new DataSourceColumnRequest(detail.getFieldName(), detail.getDataType()));
        }
    }
    return dataSourceColumnRequests;
}

Example 5 with DataSourceColumnRequest

use of com.webank.wedatasphere.qualitis.rule.request.DataSourceColumnRequest in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method constructDataSourceRequest.

private List<DataSourceRequest> constructDataSourceRequest(Set<RuleDataSource> ruleDataSources) {
    List<DataSourceRequest> dataSourceRequests = new ArrayList<>(ruleDataSources.size());
    for (RuleDataSource ruleDataSource : ruleDataSources) {
        DataSourceRequest dataSourceRequest = new DataSourceRequest();
        if (ruleDataSource.getLinkisDataSourceId() != null) {
            dataSourceRequest.setLinkisDataSourceId(ruleDataSource.getLinkisDataSourceId());
            dataSourceRequest.setLinkisDataSourceName(ruleDataSource.getLinkisDataSourceName());
            dataSourceRequest.setLinkisDataSourceType(TemplateDataSourceTypeEnum.getMessage(ruleDataSource.getDatasourceType()));
        }
        dataSourceRequest.setClusterName(ruleDataSource.getClusterName());
        dataSourceRequest.setBlackList(ruleDataSource.getBlackColName());
        dataSourceRequest.setProxyUser(ruleDataSource.getProxyUser());
        dataSourceRequest.setTableName(ruleDataSource.getTableName());
        dataSourceRequest.setDbName(ruleDataSource.getDbName());
        dataSourceRequest.setFilter(ruleDataSource.getFilter());
        String colNamesOrigin = ruleDataSource.getColName();
        List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>();
        if (!StringUtils.isBlank(colNamesOrigin)) {
            String[] colNamesSplit = colNamesOrigin.split(SpecCharEnum.VERTICAL_BAR.getValue());
            for (String str : colNamesSplit) {
                DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest();
                dataSourceColumnRequest.setColumnName(str.split(":")[0]);
                dataSourceColumnRequest.setDataType(str.split(":")[1]);
                dataSourceColumnRequests.add(dataSourceColumnRequest);
            }
        }
        dataSourceRequest.setColNames(dataSourceColumnRequests);
        dataSourceRequests.add(dataSourceRequest);
    }
    return dataSourceRequests;
}