Examples with MultiDataSourceJoinColumnRequest com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinColumnRequest used on opensource projects

Example 1 with MultiDataSourceJoinColumnRequest

use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinColumnRequest in project Qualitis by WeBankFinTech.

the class RuleBatchServiceImpl method getMultiDataSourceJoinRequest.

private void getMultiDataSourceJoinRequest(List<MultiDataSourceJoinConfigRequest> mappings, ExcelMultiTemplateRuleByProject excelMultiTemplateRule) throws UnExpectedRequestException {
    String leftStatement = excelMultiTemplateRule.getLeftMappingStatement();
    if (StringUtils.isNotBlank(leftStatement)) {
        MultiDataSourceJoinConfigRequest mapping = new MultiDataSourceJoinConfigRequest();
        List<MultiDataSourceJoinColumnRequest> leftRequest = getMultiDataSourceJoinColumnRequest(excelMultiTemplateRule.getLeftMappingNames(), excelMultiTemplateRule.getLeftMappingTypes());
        List<MultiDataSourceJoinColumnRequest> rightRequest = getMultiDataSourceJoinColumnRequest(excelMultiTemplateRule.getRightMappingNames(), excelMultiTemplateRule.getRightMappingTypes());
        mapping.setLeft(leftRequest);
        mapping.setLeftStatement(excelMultiTemplateRule.getLeftMappingStatement());
        mapping.setOperation(MappingOperationEnum.getOperationCode(excelMultiTemplateRule.getMappingOperation()));
        mapping.setRight(rightRequest);
        mapping.setRightStatement(excelMultiTemplateRule.getRightMappingStatement());
        mappings.add(mapping);
    }
}

Example 2 with MultiDataSourceJoinColumnRequest

use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinColumnRequest in project Qualitis by WeBankFinTech.

the class AutoArgumentAdapter method generateAndConcatStatement.

private String generateAndConcatStatement(TemplateMidTableInputMeta templateMidTableInputMeta, List<MultiDataSourceJoinConfigRequest> mappings, Rule rule) {
    String concatTemplate = templateMidTableInputMeta.getConcatTemplate();
    Collection<TemplateMidTableInputMeta> children = templateMidTableInputMeta.getChildren();
    List<String> originStatement = new ArrayList<>();
    if (rule.getTemplate().getMidTableAction().contains(DIFF_COUNT)) {
        MultiDataSourceJoinConfigRequest multiDataSourceJoinConfigRequest = new MultiDataSourceJoinConfigRequest();
        MultiDataSourceJoinColumnRequest leftJoinColumnRequest = new MultiDataSourceJoinColumnRequest("tmp1.".concat(DIFF_COUNT), "int");
        MultiDataSourceJoinColumnRequest rightJoinColumnRequest = new MultiDataSourceJoinColumnRequest("tmp2.".concat(DIFF_COUNT), "int");
        multiDataSourceJoinConfigRequest.setOperation(MappingOperationEnum.EQUAL.getCode());
        multiDataSourceJoinConfigRequest.setLeft(Arrays.asList(leftJoinColumnRequest));
        multiDataSourceJoinConfigRequest.setRight(Arrays.asList(rightJoinColumnRequest));
        multiDataSourceJoinConfigRequest.setLeftStatement("tmp1.".concat(DIFF_COUNT));
        multiDataSourceJoinConfigRequest.setRightStatement("tmp2.".concat(DIFF_COUNT));
        mappings.add(multiDataSourceJoinConfigRequest);
    }
    for (MultiDataSourceJoinConfigRequest mapping : mappings) {
        String replacedTemplate = concatTemplate;
        Boolean addedFlag = false;
        for (TemplateMidTableInputMeta child : children) {
            if (child.getInputType().equals(TemplateInputTypeEnum.LEFT_STATEMENT.getCode())) {
                replacedTemplate = replacedTemplate.replace(getPlaceHolder(child.getPlaceholder()), mapping.getLeftStatement());
            } else if (child.getInputType().equals(TemplateInputTypeEnum.OPERATION.getCode())) {
                replacedTemplate = replacedTemplate.replace(getPlaceHolder(child.getPlaceholder()), MappingOperationEnum.getByCode(mapping.getOperation()).getSymbol());
            } else if (child.getInputType().equals(TemplateInputTypeEnum.RIGHT_STATEMENT.getCode())) {
                replacedTemplate = replacedTemplate.replace(getPlaceHolder(child.getPlaceholder()), mapping.getRightStatement());
            } else if (child.getInputType().equals(TemplateInputTypeEnum.SOURCE_FIELD.getCode())) {
                for (MultiDataSourceJoinColumnRequest multiDataSourceJoinColumnRequest : mapping.getLeft()) {
                    String tmp1 = concatTemplate.replace(getPlaceHolder(child.getPlaceholder()), multiDataSourceJoinColumnRequest.getColumnName());
                    originStatement.add(tmp1);
                    addedFlag = true;
                }
            } else if (child.getInputType().equals(TemplateInputTypeEnum.TARGET_FIELD.getCode())) {
                for (MultiDataSourceJoinColumnRequest multiDataSourceJoinColumnRequest : mapping.getRight()) {
                    String tmp1 = concatTemplate.replace(getPlaceHolder(child.getPlaceholder()), multiDataSourceJoinColumnRequest.getColumnName());
                    originStatement.add(tmp1);
                    addedFlag = true;
                }
            }
        }
        if (!addedFlag) {
            originStatement.add(replacedTemplate);
        }
    }
    return generateStatementByConcatStr(originStatement, "AND");
}

Example 3 with MultiDataSourceJoinColumnRequest

use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinColumnRequest in project Qualitis by WeBankFinTech.

the class AddMultiRuleRequestBuilder method solveMapping.

private void solveMapping(String mappings, Map<String, List<ColumnInfoDetail>> colsMap) {
    List<MultiDataSourceJoinConfigRequest> requests = new ArrayList<>();
    String[] mappingStrs = mappings.split(AND);
    for (String currentMapping : mappingStrs) {
        List<MultiDataSourceJoinColumnRequest> left = new ArrayList<>();
        List<MultiDataSourceJoinColumnRequest> right = new ArrayList<>();
        MultiDataSourceJoinConfigRequest multiDataSourceJoinConfigRequest = new MultiDataSourceJoinConfigRequest();
        Iterator operator = OPERATION_CODE.entrySet().iterator();
        while (operator.hasNext()) {
            Entry entry = (Entry) operator.next();
            if (currentMapping.contains(entry.getKey().toString())) {
                multiDataSourceJoinConfigRequest.setOperation((Integer) entry.getValue());
                String[] statements = currentMapping.split(entry.getKey().toString());
                String leftStatement = statements[0];
                String rightStatement = statements[1];
                multiDataSourceJoinConfigRequest.setLeftStatement(leftStatement);
                multiDataSourceJoinConfigRequest.setRightStatement(rightStatement);
                // Statement to request
                parseStatementColumns(colsMap.get(TMP_1), left, leftStatement);
                parseStatementColumns(colsMap.get(TMP_2), right, rightStatement);
                multiDataSourceJoinConfigRequest.setLeft(left);
                multiDataSourceJoinConfigRequest.setRight(right);
                break;
            }
        }
        requests.add(multiDataSourceJoinConfigRequest);
    }
    addMultiSourceRuleRequest.setMappings(requests);
}

Example 4 with MultiDataSourceJoinColumnRequest

use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinColumnRequest in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method constructMultiRequest.

private AddMultiSourceRuleRequest constructMultiRequest(Rule rule, RuleGroup ruleGroup) {
    AddMultiSourceRuleRequest addMultiSourceRuleRequest = new AddMultiSourceRuleRequest();
    String newVersion = ruleGroup.getVersion();
    if (StringUtils.isEmpty(newVersion)) {
        String newRuleName = rule.getName() + "_copy_" + ruleGroup.getId();
        LOGGER.info("Multi rule start to be copied. Copied rule name: " + newRuleName);
        addMultiSourceRuleRequest.setRuleName(newRuleName);
        if (StringUtils.isNotBlank(rule.getCnName())) {
            addMultiSourceRuleRequest.setRuleCnName(rule.getCnName() + "_副本");
        }
    } else {
        String oldVersion = rule.getRuleGroup().getVersion();
        LOGGER.info("Multi rule start to be copied. Copied rule name: " + rule.getName() + "_" + ruleGroup.getVersion());
        if (StringUtils.isNotBlank(oldVersion) && rule.getName().endsWith(oldVersion)) {
            addMultiSourceRuleRequest.setRuleName(rule.getName().replace(oldVersion, newVersion));
            if (StringUtils.isNotBlank(rule.getCnName())) {
                addMultiSourceRuleRequest.setRuleCnName(rule.getCnName().replace(oldVersion, newVersion));
            }
        } else {
            addMultiSourceRuleRequest.setRuleName(rule.getName() + "_" + newVersion);
            if (StringUtils.isNotBlank(rule.getCnName())) {
                addMultiSourceRuleRequest.setRuleCnName(rule.getCnName() + "_" + newVersion);
            }
        }
    }
    String ruleDetail = rule.getDetail();
    String clusterName = rule.getRuleDataSources().iterator().next().getClusterName();
    addMultiSourceRuleRequest.setAbortOnFailure(rule.getAbortOnFailure());
    addMultiSourceRuleRequest.setClusterName(clusterName);
    addMultiSourceRuleRequest.setRuleDetail(ruleDetail);
    addMultiSourceRuleRequest.setCsId(rule.getCsId());
    addMultiSourceRuleRequest.setSpecifyStaticStartupParam(rule.getSpecifyStaticStartupParam());
    addMultiSourceRuleRequest.setDeleteFailCheckResult(rule.getDeleteFailCheckResult());
    addMultiSourceRuleRequest.setMultiSourceRuleTemplateId(rule.getTemplate().getId());
    addMultiSourceRuleRequest.setStaticStartupParam(rule.getStaticStartupParam());
    addMultiSourceRuleRequest.setProjectId(ruleGroup.getProjectId());
    addMultiSourceRuleRequest.setRuleGroupId(ruleGroup.getId());
    List<RuleVariable> filterRuleVariable = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getTemplateMidTableInputMeta().getInputType().equals(TemplateInputTypeEnum.CONDITION.getCode())).collect(Collectors.toList());
    if (filterRuleVariable != null && filterRuleVariable.size() != 0) {
        addMultiSourceRuleRequest.setFilter(filterRuleVariable.iterator().next().getValue());
    }
    // Data source request
    for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
        String databaseName = ruleDataSource.getDbName();
        String tableName = ruleDataSource.getTableName();
        String filter = ruleDataSource.getFilter();
        Integer datasourceIndex = ruleDataSource.getDatasourceIndex();
        MultiDataSourceConfigRequest dataSourceConfigRequest = new MultiDataSourceConfigRequest();
        dataSourceConfigRequest.setDbName(databaseName);
        dataSourceConfigRequest.setTableName(tableName);
        dataSourceConfigRequest.setFilter(filter);
        dataSourceConfigRequest.setProxyUser(ruleDataSource.getProxyUser());
        if (ruleDataSource.getLinkisDataSourceId() != null) {
            dataSourceConfigRequest.setLinkisDataSourceType(TemplateDataSourceTypeEnum.getMessage(ruleDataSource.getDatasourceType()));
            dataSourceConfigRequest.setLinkisDataSourceName(ruleDataSource.getLinkisDataSourceName());
            dataSourceConfigRequest.setLinkisDataSourceId(ruleDataSource.getLinkisDataSourceId());
        }
        if (datasourceIndex == 0) {
            addMultiSourceRuleRequest.setSource(dataSourceConfigRequest);
        } else {
            addMultiSourceRuleRequest.setTarget(dataSourceConfigRequest);
        }
    }
    // Mapping
    List<MultiDataSourceJoinConfigRequest> mappings = new ArrayList<>();
    for (RuleDataSourceMapping mapping : rule.getRuleDataSourceMappings()) {
        MultiDataSourceJoinConfigRequest multiDataSourceJoinConfigRequest = new MultiDataSourceJoinConfigRequest();
        multiDataSourceJoinConfigRequest.setOperation(mapping.getOperation());
        multiDataSourceJoinConfigRequest.setLeftStatement(mapping.getLeftStatement());
        multiDataSourceJoinConfigRequest.setRightStatement(mapping.getRightStatement());
        List<MultiDataSourceJoinColumnRequest> left = getMultiDataSourceJoinColumnRequest(mapping.getLeftColumnNames(), mapping.getLeftColumnTypes());
        List<MultiDataSourceJoinColumnRequest> right = getMultiDataSourceJoinColumnRequest(mapping.getRightColumnNames(), mapping.getLeftColumnTypes());
        multiDataSourceJoinConfigRequest.setLeft(left);
        multiDataSourceJoinConfigRequest.setRight(right);
        mappings.add(multiDataSourceJoinConfigRequest);
    }
    addMultiSourceRuleRequest.setMappings(mappings);
    List<AlarmConfigRequest> alarmConfigRequests = constructAlarmConfigRequest(rule.getAlarmConfigs());
    addMultiSourceRuleRequest.setAlarm(true);
    addMultiSourceRuleRequest.setAlarmVariable(alarmConfigRequests);
    return addMultiSourceRuleRequest;
}

Example 5 with MultiDataSourceJoinColumnRequest

use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinColumnRequest in project Qualitis by WeBankFinTech.

the class MetaDataServiceImpl method constructRequest.

private AddMultiSourceRuleRequest constructRequest(MulDbRequest request, FilterRequest filterRequest, String currentSourceTable, String currentTargetTable, int index, int total, String loginUser) throws UnExpectedRequestException, MetaDataAcquireFailedException, IOException {
    LOGGER.info("Start to construct add multi source rule request.");
    // rule basic info.
    AddMultiSourceRuleRequest addMultiSourceRuleRequest = new AddMultiSourceRuleRequest();
    basicInfo(addMultiSourceRuleRequest, request, index, total);
    MultiDataSourceConfigRequest sourceConfigRequest = new MultiDataSourceConfigRequest();
    sourceConfigRequest.setLinkisDataSourceName(request.getSourceLinkisDataSourceName());
    sourceConfigRequest.setLinkisDataSourceType(request.getSourceLinkisDataSourceType());
    sourceConfigRequest.setLinkisDataSourceId(request.getSourceLinkisDataSourceId());
    sourceConfigRequest.setProxyUser(request.getProxyUser());
    sourceConfigRequest.setDbName(request.getSourceDb());
    sourceConfigRequest.setContextService(false);
    MultiDataSourceConfigRequest targetConfigRequest = new MultiDataSourceConfigRequest();
    targetConfigRequest.setLinkisDataSourceName(request.getTargetLinkisDataSourceName());
    targetConfigRequest.setLinkisDataSourceType(request.getTargetLinkisDataSourceType());
    targetConfigRequest.setLinkisDataSourceId(request.getTargetLinkisDataSourceId());
    targetConfigRequest.setProxyUser(request.getProxyUser());
    targetConfigRequest.setDbName(request.getTargetDb());
    targetConfigRequest.setContextService(false);
    if (filterRequest == null) {
        // source table and target table
        sourceConfigRequest.setTableName(currentSourceTable);
        sourceConfigRequest.setFilter("true");
        targetConfigRequest.setTableName(currentTargetTable);
        targetConfigRequest.setFilter("true");
        addMultiSourceRuleRequest.setMultiSourceRuleTemplateId(20L);
        List<AlarmConfigRequest> alarmConfigRequests = new ArrayList<>();
        alarmConfigRequests.add(new AlarmConfigRequest(645L, CheckTemplateEnum.FIXED_VALUE.getCode(), CompareTypeEnum.EQUAL.getCode(), 0.0));
        addMultiSourceRuleRequest.setAlarmVariable(alarmConfigRequests);
    } else {
        sourceConfigRequest.setTableName(filterRequest.getSourceTable());
        if (StringUtils.isNotBlank(filterRequest.getSourceFilter())) {
            sourceConfigRequest.setFilter("! (" + filterRequest.getSourceFilter() + ")");
        } else {
            sourceConfigRequest.setFilter("true");
        }
        targetConfigRequest.setTableName(filterRequest.getTargetTable());
        if (StringUtils.isNotBlank(filterRequest.getTargetFilter())) {
            targetConfigRequest.setFilter("! (" + filterRequest.getTargetFilter() + ")");
        } else {
            targetConfigRequest.setFilter("true");
        }
        addMultiSourceRuleRequest.setMultiSourceRuleTemplateId(17L);
        List<AlarmConfigRequest> alarmConfigRequests = new ArrayList<>();
        alarmConfigRequests.add(new AlarmConfigRequest(33L, CheckTemplateEnum.FIXED_VALUE.getCode(), CompareTypeEnum.EQUAL.getCode(), 0.0));
        addMultiSourceRuleRequest.setAlarmVariable(alarmConfigRequests);
    }
    addMultiSourceRuleRequest.setSource(sourceConfigRequest);
    addMultiSourceRuleRequest.setTarget(targetConfigRequest);
    // All fields mappings
    LOGGER.info("Start to get all fields with source database's table and target database's table.");
    List<ColumnInfoDetail> sourceFields;
    List<ColumnInfoDetail> targetFields;
    if (request.getSourceLinkisDataSourceId() == null) {
        GetUserColumnByTableIdRequest getUserColumnSourceRequest = new GetUserColumnByTableIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getSourceDb(), filterRequest == null ? currentSourceTable : filterRequest.getSourceTable());
        getUserColumnSourceRequest.setProxyUser(request.getProxyUser());
        getUserColumnSourceRequest.setLoginUser(loginUser);
        sourceFields = getUserColumnByTableId(getUserColumnSourceRequest).getData().getData();
    } else {
        sourceFields = getColumnsByDataSource(request.getClusterName(), request.getProxyUser(), request.getSourceLinkisDataSourceId(), request.getSourceDb(), currentSourceTable).getData().getData();
    }
    if (request.getTargetLinkisDataSourceId() == null) {
        GetUserColumnByTableIdRequest getUserColumnTargetRequest = new GetUserColumnByTableIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getTargetDb(), filterRequest == null ? currentTargetTable : filterRequest.getTargetTable());
        getUserColumnTargetRequest.setProxyUser(request.getProxyUser());
        getUserColumnTargetRequest.setLoginUser(loginUser);
        targetFields = getUserColumnByTableId(getUserColumnTargetRequest).getData().getData();
    } else {
        targetFields = getColumnsByDataSource(request.getClusterName(), request.getProxyUser(), request.getTargetLinkisDataSourceId(), request.getTargetDb(), currentTargetTable).getData().getData();
    }
    if (CollectionUtils.isEmpty(sourceFields) || CollectionUtils.isEmpty(targetFields)) {
        throw new UnExpectedRequestException("There is table which has none field");
    }
    LOGGER.info("Success to get all fields with source database's table and target database's table.");
    // Sorted columns.
    Collections.sort(sourceFields, new Comparator<ColumnInfoDetail>() {

        @Override
        public int compare(ColumnInfoDetail front, ColumnInfoDetail back) {
            return front.getFieldName().compareTo(back.getFieldName());
        }
    });
    Collections.sort(targetFields, new Comparator<ColumnInfoDetail>() {

        @Override
        public int compare(ColumnInfoDetail front, ColumnInfoDetail back) {
            return front.getFieldName().compareTo(back.getFieldName());
        }
    });
    String sourceFieldStr = sourceFields.stream().map(ColumnInfoDetail::getFieldName).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
    String sourceFieldTypeStr = sourceFields.stream().map(ColumnInfoDetail::getDataType).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
    String targetFieldStr = targetFields.stream().map(ColumnInfoDetail::getFieldName).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
    String targetFieldTypeStr = targetFields.stream().map(ColumnInfoDetail::getDataType).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
    if (sourceFields.size() == 0 || targetFields.size() == 0 || sourceFields.size() != targetFields.size() || !sourceFieldStr.equals(targetFieldStr) || !sourceFieldTypeStr.equals(targetFieldTypeStr)) {
        throw new UnExpectedRequestException("Create multi-db rules failed, because the fields' name of table is different.");
    }
    List<MultiDataSourceJoinConfigRequest> mappings = new ArrayList<>(sourceFields.size());
    for (int j = 0; j < sourceFields.size() && j < targetFields.size(); j++) {
        ColumnInfoDetail currentSourceField = sourceFields.get(j);
        ColumnInfoDetail currentTargetField = targetFields.get(j);
        if (filterRequest != null && CollectionUtils.isNotEmpty(filterRequest.getFilterColumnList()) && filterRequest.getFilterColumnList().contains(currentSourceField.getFieldName())) {
            continue;
        }
        if (currentSourceField.getFieldName().equals(currentTargetField.getFieldName()) && currentSourceField.getDataType().equals(currentTargetField.getDataType())) {
            // TODO: continue
            MultiDataSourceJoinConfigRequest multiDataSourceJoinConfigRequest = new MultiDataSourceJoinConfigRequest();
            MultiDataSourceJoinColumnRequest leftJoinColumnRequest = new MultiDataSourceJoinColumnRequest("tmp1.".concat(currentSourceField.getFieldName()), currentSourceField.getDataType());
            MultiDataSourceJoinColumnRequest rightJoinColumnRequest = new MultiDataSourceJoinColumnRequest("tmp2.".concat(currentTargetField.getFieldName()), currentTargetField.getDataType());
            multiDataSourceJoinConfigRequest.setOperation(MappingOperationEnum.EQUAL.getCode());
            multiDataSourceJoinConfigRequest.setLeft(Arrays.asList(leftJoinColumnRequest));
            multiDataSourceJoinConfigRequest.setRight(Arrays.asList(rightJoinColumnRequest));
            multiDataSourceJoinConfigRequest.setLeftStatement("tmp1.".concat(currentSourceField.getFieldName()));
            multiDataSourceJoinConfigRequest.setRightStatement("tmp2.".concat(currentTargetField.getFieldName()));
            mappings.add(multiDataSourceJoinConfigRequest);
        } else {
            throw new UnExpectedRequestException("Create multi-db rules failed, because the fields' name of tables is different");
        }
    }
    addMultiSourceRuleRequest.setMappings(mappings);
    addMultiSourceRuleRequest.setLoginUser(loginUser);
    LOGGER.info("Success to construct add multi source rule request. Request[{}]", new ObjectMapper().writeValueAsString(addMultiSourceRuleRequest));
    return addMultiSourceRuleRequest;
}