Examples with DataSourceRequest com.webank.wedatasphere.qualitis.rule.request.DataSourceRequest used on opensource projects

Example 1 with DataSourceRequest

use of com.webank.wedatasphere.qualitis.rule.request.DataSourceRequest in project Qualitis by WeBankFinTech.

the class AddFileRuleRequestBuilder method solveDatasource.

private void solveDatasource(String datasource) throws UnExpectedRequestException {
    List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>();
    DataSourceRequest dataSourceRequest = new DataSourceRequest();
    String clusterName;
    String database;
    String table;
    String filter;
    String[] datasourceStrs = datasource.split(SpecCharEnum.COLON.getValue());
    if (datasourceStrs.length > TWO) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    String[] datasources = datasourceStrs[0].split(SpecCharEnum.PERIOD.getValue());
    filter = datasourceStrs.length >= TWO ? datasourceStrs[1] : "";
    if (datasources.length != FOUR && datasources.length != THREE) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    clusterName = datasources[0];
    database = datasources[1];
    table = datasources[2];
    dataSourceRequest.setClusterName(clusterName);
    dataSourceRequest.setDbName(database);
    dataSourceRequest.setTableName(table);
    dataSourceRequest.setColNames(dataSourceColumnRequests);
    dataSourceRequest.setFilter(filter);
    dataSourceRequest.setProxyUser(userName);
    this.addFileRuleRequest.setDatasource(dataSourceRequest);
}

Example 2 with DataSourceRequest

use of com.webank.wedatasphere.qualitis.rule.request.DataSourceRequest in project Qualitis by WeBankFinTech.

the class MultiSourceRuleServiceImpl method generateDataSourceRequest.

private List<DataSourceRequest> generateDataSourceRequest(String clusterName, MultiDataSourceConfigRequest sourceConfig, MultiDataSourceConfigRequest targetConfig) {
    List<DataSourceRequest> dataSourceRequests = new ArrayList<>();
    DataSourceRequest sourceDataSourceRequest = new DataSourceRequest();
    sourceDataSourceRequest.setClusterName(clusterName);
    sourceDataSourceRequest.setFilter(sourceConfig.getFilter());
    sourceDataSourceRequest.setTableName(sourceConfig.getTableName());
    sourceDataSourceRequest.setDbName(sourceConfig.getDbName());
    sourceDataSourceRequest.setColNames(new ArrayList<>());
    sourceDataSourceRequest.setDatasourceIndex(0);
    sourceDataSourceRequest.setFileId(sourceConfig.getFileId());
    sourceDataSourceRequest.setFileTablesDesc(sourceConfig.getFileTableDesc());
    sourceDataSourceRequest.setFileDelimiter(sourceConfig.getFileDelimiter());
    sourceDataSourceRequest.setFileType(sourceConfig.getFileType());
    sourceDataSourceRequest.setFileHeader(sourceConfig.getFileHeader());
    sourceDataSourceRequest.setProxyUser(sourceConfig.getProxyUser());
    sourceDataSourceRequest.setFileHashValues(sourceConfig.getFileHashValues());
    sourceDataSourceRequest.setLinkisDataSourceId(sourceConfig.getLinkisDataSourceId());
    sourceDataSourceRequest.setLinkisDataSourceName(sourceConfig.getLinkisDataSourceName());
    sourceDataSourceRequest.setLinkisDataSourceType(sourceConfig.getLinkisDataSourceType());
    sourceDataSourceRequest.setLinkisDataSourceVersionId(sourceConfig.getLinkisDataSourceVersionId());
    DataSourceRequest targetDataSourceRequest = new DataSourceRequest();
    targetDataSourceRequest.setClusterName(clusterName);
    targetDataSourceRequest.setFilter(targetConfig.getFilter());
    targetDataSourceRequest.setTableName(targetConfig.getTableName());
    targetDataSourceRequest.setDbName(targetConfig.getDbName());
    targetDataSourceRequest.setColNames(new ArrayList<>());
    targetDataSourceRequest.setDatasourceIndex(1);
    targetDataSourceRequest.setFileId(targetConfig.getFileId());
    targetDataSourceRequest.setFileTablesDesc(targetConfig.getFileTableDesc());
    targetDataSourceRequest.setFileDelimiter(targetConfig.getFileDelimiter());
    targetDataSourceRequest.setFileType(targetConfig.getFileType());
    targetDataSourceRequest.setFileHeader(targetConfig.getFileHeader());
    targetDataSourceRequest.setProxyUser(targetConfig.getProxyUser());
    targetDataSourceRequest.setFileHashValues(targetConfig.getFileHashValues());
    targetDataSourceRequest.setLinkisDataSourceId(targetConfig.getLinkisDataSourceId());
    targetDataSourceRequest.setLinkisDataSourceName(targetConfig.getLinkisDataSourceName());
    targetDataSourceRequest.setLinkisDataSourceType(targetConfig.getLinkisDataSourceType());
    targetDataSourceRequest.setLinkisDataSourceVersionId(targetConfig.getLinkisDataSourceVersionId());
    dataSourceRequests.add(sourceDataSourceRequest);
    dataSourceRequests.add(targetDataSourceRequest);
    return dataSourceRequests;
}

Example 3 with DataSourceRequest

use of com.webank.wedatasphere.qualitis.rule.request.DataSourceRequest in project Qualitis by WeBankFinTech.

the class AddRuleRequestBuilder method solveDatasource.

private void solveDatasource(String datasource) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    List<DataSourceRequest> dataSourceRequests = new ArrayList<>(1);
    List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>(1);
    DataSourceRequest dataSourceRequest = new DataSourceRequest();
    String clusterName;
    String database;
    String table;
    String col;
    String filter;
    String[] datasourceStrs = datasource.split(SpecCharEnum.COLON.getValue());
    if (datasourceStrs.length > TWO) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    String dbAndTable = datasourceStrs[0];
    String dataSourceId = "";
    Matcher matcherId = DATA_SOURCE_ID.matcher(dbAndTable.toUpperCase());
    String dataSourceName = "";
    Matcher matcherName = DATA_SOURCE_NAME.matcher(dbAndTable.toUpperCase());
    while (matcherId.find()) {
        String group = matcherId.group();
        dataSourceId = group.replace(".(", "").replace(")", "").split("=")[1];
        int startIndex = dbAndTable.toUpperCase().indexOf(group);
        String replaceStr = dbAndTable.substring(startIndex, startIndex + group.length());
        dbAndTable = dbAndTable.replace(replaceStr, "");
    }
    if (StringUtils.isBlank(dataSourceId)) {
        while (matcherName.find()) {
            String group = matcherName.group();
            int startIndex = dbAndTable.toUpperCase().indexOf(group);
            String replaceStr = dbAndTable.substring(startIndex, startIndex + group.length());
            dataSourceName = replaceStr.replace(".(", "").replace(")", "").split("=")[1];
            dbAndTable = dbAndTable.replace(replaceStr, "");
        }
    }
    String[] datasources = dbAndTable.split(SpecCharEnum.PERIOD.getValue());
    filter = datasourceStrs[1];
    if (datasources.length != FOUR && datasources.length != THREE) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    clusterName = datasources[0];
    database = datasources[1];
    table = datasources[2];
    col = datasources.length >= FOUR ? datasources[3] : "";
    List<ColumnInfoDetail> cols = metaDataClient.getColumnInfo(clusterName, database, table, userName);
    if (StringUtils.isNotBlank(dataSourceId)) {
        LOGGER.info("Find data source connect. Data source ID: " + dataSourceId);
        dataSourceRequest.setLinkisDataSourceId(Long.parseLong(dataSourceId));
        GeneralResponse<Map> response = metaDataClient.getDataSourceInfoDetail(clusterName, userName, dataSourceRequest.getLinkisDataSourceId(), null);
        cols = metaDataClient.getColumnsByDataSource(clusterName, userName, dataSourceRequest.getLinkisDataSourceId(), database, table).getContent();
        Map dataSourceInfo = ((Map) response.getData().get("info"));
        String dataSourceInfoName = (String) dataSourceInfo.get("dataSourceName");
        String dataSourceInfoType = (String) ((Map) dataSourceInfo.get("dataSourceType")).get("name");
        dataSourceRequest.setLinkisDataSourceName(dataSourceInfoName);
        dataSourceRequest.setLinkisDataSourceType(dataSourceInfoType);
    } else if (StringUtils.isNotBlank(dataSourceName)) {
        LOGGER.info("Find data source connect. Data source name: " + dataSourceName);
        GeneralResponse<Map> response = metaDataClient.getDataSourceInfoDetailByName(clusterName, userName, dataSourceName);
        Map dataSourceInfo = ((Map) response.getData().get("info"));
        String dataSourceInfoName = (String) dataSourceInfo.get("dataSourceName");
        String dataSourceInfoType = (String) ((Map) dataSourceInfo.get("dataSourceType")).get("name");
        Integer currentDataSourceId = (Integer) dataSourceInfo.get("id");
        dataSourceRequest.setLinkisDataSourceId(currentDataSourceId.longValue());
        cols = metaDataClient.getColumnsByDataSource(clusterName, userName, currentDataSourceId.longValue(), database, table).getContent();
        dataSourceRequest.setLinkisDataSourceName(dataSourceInfoName);
        dataSourceRequest.setLinkisDataSourceType(dataSourceInfoType);
    }
    // For one or more fields
    if (StringUtils.isBlank(col)) {
        LOGGER.info("Table count check.");
    } else {
        boolean blackList = false;
        List<String> colsInDatasource = new ArrayList<>();
        if (col.startsWith(SpecCharEnum.MINUS.getValue())) {
            blackList = true;
            col = col.replace(SpecCharEnum.MINUS.getValue(), "");
        }
        if (col.contains(SpecCharEnum.LEFT_BRACKET.getValue()) && col.contains(SpecCharEnum.RIGHT_BRACKET.getValue())) {
            col = col.replace(SpecCharEnum.LEFT_BRACKET.getValue(), "").replace(SpecCharEnum.RIGHT_BRACKET.getValue(), "");
            for (String currentCol : col.split(SpecCharEnum.COMMA.getValue())) {
                colsInDatasource.add(currentCol);
            }
        } else {
            colsInDatasource.add(col);
        }
        if (blackList) {
            for (ColumnInfoDetail columnInfoDetail : cols) {
                if (colsInDatasource.contains(columnInfoDetail.getFieldName())) {
                    continue;
                } else {
                    DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest(columnInfoDetail.getFieldName(), columnInfoDetail.getDataType());
                    dataSourceColumnRequests.add(dataSourceColumnRequest);
                }
            }
        } else {
            for (String colName : colsInDatasource) {
                String type = cols.stream().filter(field -> field.getFieldName().equals(colName)).map(ColumnInfoDetail::getDataType).collect(Collectors.joining());
                DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest(colName, type);
                dataSourceColumnRequests.add(dataSourceColumnRequest);
            }
        }
        dataSourceRequest.setBlackList(blackList);
    }
    dataSourceRequest.setClusterName(clusterName);
    dataSourceRequest.setDbName(database);
    dataSourceRequest.setTableName(table);
    dataSourceRequest.setColNames(dataSourceColumnRequests);
    dataSourceRequest.setFilter(filter);
    dataSourceRequest.setProxyUser(userName);
    dataSourceRequests.add(dataSourceRequest);
    this.addRuleRequest.setDatasource(dataSourceRequests);
}

Example 4 with DataSourceRequest

use of com.webank.wedatasphere.qualitis.rule.request.DataSourceRequest in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method constructFileRequest.

private AddFileRuleRequest constructFileRequest(Rule rule, RuleGroup ruleGroup) {
    AddFileRuleRequest addFileRuleRequest = new AddFileRuleRequest();
    String newVersion = ruleGroup.getVersion();
    if (StringUtils.isEmpty(newVersion)) {
        String newRuleName = rule.getName() + "_copy_" + ruleGroup.getId();
        LOGGER.info("File rule start to be copied. Copied rule name: " + newRuleName);
        addFileRuleRequest.setRuleName(newRuleName);
        if (StringUtils.isNotBlank(rule.getCnName())) {
            addFileRuleRequest.setRuleCnName(rule.getCnName() + "_副本");
        }
    } else {
        String oldVersion = rule.getRuleGroup().getVersion();
        LOGGER.info("File rule start to be copied. Copied rule name: " + rule.getName() + "_" + ruleGroup.getVersion());
        if (StringUtils.isNotBlank(oldVersion) && rule.getName().endsWith(oldVersion)) {
            addFileRuleRequest.setRuleName(rule.getName().replace(oldVersion, newVersion));
            if (StringUtils.isNotBlank(rule.getCnName())) {
                addFileRuleRequest.setRuleCnName(rule.getCnName().replace(oldVersion, newVersion));
            }
        } else {
            addFileRuleRequest.setRuleName(rule.getName() + "_" + newVersion);
            if (StringUtils.isNotBlank(rule.getCnName())) {
                addFileRuleRequest.setRuleCnName(rule.getCnName() + "_" + newVersion);
            }
        }
    }
    String ruleDetail = rule.getDetail();
    addFileRuleRequest.setRuleDetail(ruleDetail);
    addFileRuleRequest.setAbortOnFailure(rule.getAbortOnFailure());
    addFileRuleRequest.setCsId(rule.getCsId());
    addFileRuleRequest.setRuleGroupId(ruleGroup.getId());
    addFileRuleRequest.setProjectId(ruleGroup.getProjectId());
    addFileRuleRequest.setAbortOnFailure(rule.getAbortOnFailure());
    for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
        DataSourceRequest dataSourceRequest = new DataSourceRequest();
        String clusterName = ruleDataSource.getClusterName();
        String databaseName = ruleDataSource.getDbName();
        String tableName = ruleDataSource.getTableName();
        String filter = ruleDataSource.getFilter();
        dataSourceRequest.setClusterName(clusterName);
        dataSourceRequest.setDbName(databaseName);
        dataSourceRequest.setTableName(tableName);
        dataSourceRequest.setFilter(filter);
        addFileRuleRequest.setDatasource(dataSourceRequest);
        break;
    }
    List<FileAlarmConfigRequest> alarmVariable = new ArrayList<>();
    for (AlarmConfig alarmConfig : rule.getAlarmConfigs()) {
        FileAlarmConfigRequest fileAlarmConfigRequest = new FileAlarmConfigRequest();
        Double threshold = alarmConfig.getThreshold();
        Integer unit = alarmConfig.getFileOutputUnit();
        Integer alarmCompareType = alarmConfig.getCompareType();
        Integer alarmOutputName = alarmConfig.getFileOutputName();
        Integer checkTemplateName = alarmConfig.getCheckTemplate();
        fileAlarmConfigRequest.setFileOutputUnit(unit);
        fileAlarmConfigRequest.setCompareType(alarmCompareType);
        fileAlarmConfigRequest.setFileOutputName(alarmOutputName);
        fileAlarmConfigRequest.setCheckTemplate(checkTemplateName);
        fileAlarmConfigRequest.setThreshold(threshold);
        RuleMetric ruleMetric = alarmConfig.getRuleMetric();
        // Recod rule metric info (unique code).
        if (ruleMetric != null) {
            String enCode = ruleMetric.getEnCode();
            fileAlarmConfigRequest.setRuleMetricEnCode(enCode);
            fileAlarmConfigRequest.setUploadAbnormalValue(alarmConfig.getUploadAbnormalValue());
            fileAlarmConfigRequest.setUploadRuleMetricValue(alarmConfig.getUploadRuleMetricValue());
        }
        fileAlarmConfigRequest.setDeleteFailCheckResult(alarmConfig.getDeleteFailCheckResult());
        alarmVariable.add(fileAlarmConfigRequest);
    }
    addFileRuleRequest.setAlarm(true);
    addFileRuleRequest.setAlarmVariable(alarmVariable);
    return addFileRuleRequest;
}

Example 5 with DataSourceRequest

use of com.webank.wedatasphere.qualitis.rule.request.DataSourceRequest in project Qualitis by WeBankFinTech.

the class RuleBatchServiceImpl method getDataSourceRequest.

private void getDataSourceRequest(List<DataSourceRequest> dataSourceRequests, ExcelTemplateRuleByProject excelTemplateRule) {
    String clusterName = excelTemplateRule.getCluster();
    if (!StringUtils.isBlank(clusterName)) {
        DataSourceRequest dataSourceRequest = new DataSourceRequest();
        if (excelTemplateRule.getLinkisDataSourceId() != null) {
            dataSourceRequest.setLinkisDataSourceId(Long.parseLong(excelTemplateRule.getLinkisDataSourceId()));
            dataSourceRequest.setLinkisDataSourceName(excelTemplateRule.getLinkisDataSourceName());
            dataSourceRequest.setLinkisDataSourceType(excelTemplateRule.getLinkisDataSourceType());
        }
        dataSourceRequest.setClusterName(excelTemplateRule.getCluster());
        dataSourceRequest.setProxyUser(excelTemplateRule.getProxyUser());
        dataSourceRequest.setTableName(excelTemplateRule.getTableName());
        dataSourceRequest.setDbName(excelTemplateRule.getDbName());
        dataSourceRequest.setFilter(excelTemplateRule.getFilter());
        String colNamesOrigin = excelTemplateRule.getColumnNames();
        List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>();
        if (!StringUtils.isBlank(colNamesOrigin)) {
            String[] colNamesSplit = colNamesOrigin.split(SpecCharEnum.VERTICAL_BAR.getValue());
            for (String str : colNamesSplit) {
                DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest();
                dataSourceColumnRequest.setColumnName(str.split(":")[0]);
                dataSourceColumnRequest.setDataType(str.split(":")[1]);
                dataSourceColumnRequests.add(dataSourceColumnRequest);
            }
        }
        // Default all column will be white list field to be imported.
        dataSourceRequest.setBlackList(false);
        dataSourceRequest.setColNames(dataSourceColumnRequests);
        dataSourceRequests.add(dataSourceRequest);
    }
}