Examples with MultiDataSourceJoinConfigRequest com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinConfigRequest used on opensource projects

Example 6 with MultiDataSourceJoinConfigRequest

use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinConfigRequest in project Qualitis by WeBankFinTech.

the class AutoArgumentAdapter method findValueFromMultiDataSourceConfig.

private Map<String, String> findValueFromMultiDataSourceConfig(TemplateMidTableInputMeta templateMidTableInputMeta, String clusterName, MultiDataSourceConfigRequest firstDataSource, MultiDataSourceConfigRequest secondDataSource, List<MultiDataSourceJoinConfigRequest> mappings) {
    Map<String, String> map = new HashMap<>(4);
    String firstFileType = firstDataSource.getFileType();
    String firstDbName = firstDataSource.getDbName();
    if (firstFileType != null && !EXCEL_TYPE.contains(firstFileType)) {
        firstDbName = "";
    }
    String secondFileType = secondDataSource.getFileType();
    String secondDbName = secondDataSource.getDbName();
    if (secondFileType != null && !EXCEL_TYPE.contains(secondFileType)) {
        secondDbName = "";
    }
    Integer inputType = templateMidTableInputMeta.getInputType();
    if (inputType.equals(TemplateInputTypeEnum.SOURCE_DB.getCode())) {
        map.put("clusterName", clusterName);
        map.put("dbName", firstDbName);
        map.put("value", firstDbName);
    } else if (inputType.equals(TemplateInputTypeEnum.SOURCE_TABLE.getCode())) {
        map.put("clusterName", clusterName);
        map.put("dbName", firstDbName);
        map.put("tableName", firstDataSource.getTableName());
        map.put("value", firstDataSource.getTableName());
    } else if (inputType.equals(TemplateInputTypeEnum.SOURCE_FIELDS.getCode())) {
        List<String> fieldName = new ArrayList<>();
        for (MultiDataSourceJoinConfigRequest request : mappings) {
            for (MultiDataSourceJoinColumnRequest columnRequest : request.getLeft()) {
                if (columnRequest.getColumnName().equals(DIFF_COUNT)) {
                    continue;
                }
                fieldName.add(columnRequest.getColumnName().replace("tmp1.", ""));
            }
        }
        map.put("value", String.join(", ", fieldName));
    } else if (inputType.equals(TemplateInputTypeEnum.TARGET_DB.getCode())) {
        map.put("clusterName", clusterName);
        map.put("dbName", secondDbName);
        map.put("value", secondDbName);
    } else if (inputType.equals(TemplateInputTypeEnum.TARGET_TABLE.getCode())) {
        map.put("clusterName", clusterName);
        map.put("dbName", secondDbName);
        map.put("tableName", secondDataSource.getTableName());
        map.put("value", secondDataSource.getTableName());
    } else if (inputType.equals(TemplateInputTypeEnum.TARGET_FIELDS.getCode())) {
        List<String> fieldName = new ArrayList<>();
        for (MultiDataSourceJoinConfigRequest request : mappings) {
            for (MultiDataSourceJoinColumnRequest columnRequest : request.getRight()) {
                if (columnRequest.getColumnName().equals(DIFF_COUNT)) {
                    continue;
                }
                fieldName.add(columnRequest.getColumnName().replace("tmp2.", ""));
            }
        }
        map.put("value", String.join(", ", fieldName));
    }
    return map;
}

Example 7 with MultiDataSourceJoinConfigRequest

use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinConfigRequest in project Qualitis by WeBankFinTech.

the class AddMultiRuleRequestBuilder method solveDatasource.

private Map<String, List<ColumnInfoDetail>> solveDatasource(StringBuffer cluster, String datasource) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    MultiDataSourceConfigRequest source;
    MultiDataSourceConfigRequest target;
    List<MultiDataSourceJoinConfigRequest> mappings = new ArrayList<>();
    String[] datasourceStrs = datasource.split(SpecCharEnum.VERTICAL_BAR.getValue());
    String sourcStr = datasourceStrs[0];
    String targetStr = datasourceStrs[1];
    Matcher matcherName = DATA_SOURCE_NAME.matcher(cluster.toString().toUpperCase());
    Matcher matcherId = DATA_SOURCE_ID.matcher(cluster.toString().toUpperCase());
    String dataSourceName = "";
    String dataSourceId = "";
    while (matcherId.find()) {
        String group = matcherId.group();
        dataSourceId = group.replace(".(", "").replace(")", "").split("=")[1];
        int startIndex = cluster.toString().toUpperCase().indexOf(group);
        String replaceStr = cluster.substring(startIndex, startIndex + group.length());
        cluster.delete(startIndex, startIndex + group.length());
    }
    if (StringUtils.isBlank(dataSourceId)) {
        while (matcherName.find()) {
            String group = matcherName.group();
            int startIndex = cluster.toString().toUpperCase().indexOf(group);
            String replaceStr = cluster.substring(startIndex, startIndex + group.length());
            dataSourceName = replaceStr.replace(".(", "").replace(")", "").split("=")[1];
            cluster.delete(startIndex, startIndex + group.length());
        }
    }
    source = toRequest(cluster.toString(), sourcStr, dataSourceId, dataSourceName);
    target = toRequest(cluster.toString(), targetStr, dataSourceId, dataSourceName);
    addMultiSourceRuleRequest.setSource(source);
    addMultiSourceRuleRequest.setTarget(target);
    addMultiSourceRuleRequest.setMappings(mappings);
    if (datasourceStrs.length > TWO) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    List<ColumnInfoDetail> leftCols = metaDataClient.getColumnInfo(cluster.toString(), source.getDbName(), source.getTableName(), userName);
    List<ColumnInfoDetail> rightCols = metaDataClient.getColumnInfo(cluster.toString(), target.getDbName(), target.getTableName(), userName);
    Map<String, List<ColumnInfoDetail>> colsMap = new HashMap<>(2);
    colsMap.put(TMP_1, leftCols);
    colsMap.put(TMP_2, rightCols);
    return colsMap;
}

Example 8 with MultiDataSourceJoinConfigRequest

use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinConfigRequest in project Qualitis by WeBankFinTech.

the class RuleBatchServiceImpl method constructAddMultiSourceRuleRequest.

private List<AddMultiSourceRuleRequest> constructAddMultiSourceRuleRequest(Map<String, List<ExcelMultiTemplateRuleByProject>> multiRulePartitionedByRuleName, Project project, String localeStr) throws UnExpectedRequestException {
    List<AddMultiSourceRuleRequest> addMultiSourceRuleRequests = new ArrayList<>();
    for (String ruleName : multiRulePartitionedByRuleName.keySet()) {
        List<ExcelMultiTemplateRuleByProject> ruleInfos = multiRulePartitionedByRuleName.get(ruleName);
        ExcelMultiTemplateRuleByProject firstCommonInfo = ruleInfos.get(0);
        AddMultiSourceRuleRequest addMultiSourceRuleRequest = new AddMultiSourceRuleRequest();
        String ruleTemplateName = firstCommonInfo.getTemplateName();
        String ruleGroupName = firstCommonInfo.getRuleGroupName();
        addMultiSourceRuleRequest.setSpecifyStaticStartupParam(firstCommonInfo.getSpecifyStaticStartupParam());
        addMultiSourceRuleRequest.setDeleteFailCheckResult(firstCommonInfo.getDeleteFailCheckResult());
        addMultiSourceRuleRequest.setStaticStartupParam(firstCommonInfo.getStaticStartupParam());
        addMultiSourceRuleRequest.setAbortOnFailure(firstCommonInfo.getAbortOnFailure());
        if (StringUtils.isBlank(ruleGroupName)) {
            throw new UnExpectedRequestException("RuleGroupName {&CAN_NOT_BE_NULL_OR_EMPTY}");
        }
        Template template = findTemplateByName(ruleTemplateName);
        if (template == null) {
            throw new UnExpectedRequestException("{&TEMPLATE_NAME}: [" + ruleTemplateName + "] {&DOES_NOT_EXIST}");
        }
        String filter = null;
        boolean alarm = false;
        String clusterName = firstCommonInfo.getClusterName();
        MultiDataSourceConfigRequest sourceConfigRequest = new MultiDataSourceConfigRequest();
        MultiDataSourceConfigRequest targetConfigRequest = new MultiDataSourceConfigRequest();
        List<MultiDataSourceJoinConfigRequest> mappings = new ArrayList<>();
        List<AlarmConfigRequest> alarmConfigRequests = new ArrayList<>();
        for (ExcelMultiTemplateRuleByProject excelMultiTemplateRule : ruleInfos) {
            if (StringUtils.isNotBlank(excelMultiTemplateRule.getWhereFilter())) {
                filter = excelMultiTemplateRule.getWhereFilter();
            }
            getAlarmConfig(alarmConfigRequests, excelMultiTemplateRule, template, localeStr);
            getMultiDataSourceRequest(sourceConfigRequest, excelMultiTemplateRule, 0);
            getMultiDataSourceRequest(targetConfigRequest, excelMultiTemplateRule, 1);
            getMultiDataSourceJoinRequest(mappings, excelMultiTemplateRule);
        }
        if (alarmConfigRequests.size() != 0) {
            alarm = true;
        }
        RuleGroup ruleGroupInDb = ruleGroupDao.findByRuleGroupNameAndProjectId(ruleGroupName, project.getId());
        if (ruleGroupInDb != null) {
            addMultiSourceRuleRequest.setRuleGroupId(ruleGroupInDb.getId());
        } else {
            RuleGroup ruleGroup = ruleGroupDao.saveRuleGroup(new RuleGroup(ruleGroupName, project.getId()));
            addMultiSourceRuleRequest.setRuleGroupId(ruleGroup.getId());
        }
        addMultiSourceRuleRequest.setRuleName(ruleName);
        addMultiSourceRuleRequest.setClusterName(clusterName);
        addMultiSourceRuleRequest.setProjectId(project.getId());
        addMultiSourceRuleRequest.setRuleDetail(firstCommonInfo.getRuleDetail());
        addMultiSourceRuleRequest.setMultiSourceRuleTemplateId(template.getId());
        addMultiSourceRuleRequest.setAlarmVariable(alarmConfigRequests);
        addMultiSourceRuleRequests.add(addMultiSourceRuleRequest);
        addMultiSourceRuleRequest.setSource(sourceConfigRequest);
        addMultiSourceRuleRequest.setTarget(targetConfigRequest);
        addMultiSourceRuleRequest.setMappings(mappings);
        addMultiSourceRuleRequest.setFilter(filter);
        addMultiSourceRuleRequest.setAlarm(alarm);
    }
    return addMultiSourceRuleRequests;
}

Example 9 with MultiDataSourceJoinConfigRequest

use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinConfigRequest in project Qualitis by WeBankFinTech.

the class RuleDataSourceMappingServiceImpl method checkAndSaveRuleDataSourceMapping.

@Override
@Transactional(rollbackFor = { Exception.class })
public List<RuleDataSourceMapping> checkAndSaveRuleDataSourceMapping(List<MultiDataSourceJoinConfigRequest> mappings, Rule rule) {
    List<RuleDataSourceMapping> result = new ArrayList<>();
    for (MultiDataSourceJoinConfigRequest mapping : mappings) {
        RuleDataSourceMapping ruleDataSourceMapping = new RuleDataSourceMapping();
        ruleDataSourceMapping.setLeftStatement(mapping.getLeftStatement());
        ruleDataSourceMapping.setRightStatement(mapping.getRightStatement());
        ruleDataSourceMapping.setOperation(mapping.getOperation());
        ruleDataSourceMapping.setLeftColumnNames(String.join(",", mapping.getLeft().stream().map(MultiDataSourceJoinColumnRequest::getColumnName).collect(Collectors.toList())));
        ruleDataSourceMapping.setLeftColumnTypes(String.join("|", mapping.getLeft().stream().map(MultiDataSourceJoinColumnRequest::getColumnType).collect(Collectors.toList())));
        ruleDataSourceMapping.setRightColumnNames(String.join(",", mapping.getRight().stream().map(MultiDataSourceJoinColumnRequest::getColumnName).collect(Collectors.toList())));
        ruleDataSourceMapping.setRightColumnTypes(String.join("|", mapping.getRight().stream().map(MultiDataSourceJoinColumnRequest::getColumnType).collect(Collectors.toList())));
        ruleDataSourceMapping.setRule(rule);
        RuleDataSourceMapping savedRuleDataSourceMapping = ruleDataSourceMappingDao.saveRuleDataSourceMapping(ruleDataSourceMapping);
        result.add(savedRuleDataSourceMapping);
    }
    return result;
}