Examples with RuleDataSource com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource used on opensource projects

Example 21 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method catchAndSolve.

private void catchAndSolve(Exception e, Integer commentCode, Integer statusCode, List<Rule> rules, Application newApplication) {
    LOGGER.error(e.getMessage(), e);
    newApplication.setApplicationComment(commentCode);
    newApplication.setStatus(statusCode);
    // Record rules info and datasource info.
    StringBuffer info = new StringBuffer();
    newApplication.setFinishTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
    for (Rule rule : rules) {
        info.append("Rule[").append(rule.getName()).append("; ");
        info.append("datasource: ").append(rule.getRuleDataSources().stream().filter(ruleDataSource -> StringUtils.isNotBlank(ruleDataSource.getDbName()) && StringUtils.isNotBlank(ruleDataSource.getTableName())).map(ruleDataSource -> ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + " ").collect(Collectors.joining()));
    }
    newApplication.setExceptionMessage(info.append("]\n").append(ExceptionUtils.getStackTrace(e)).toString());
    if (CollectionUtils.isNotEmpty(rules)) {
        Project currentProject = rules.iterator().next().getProject();
        newApplication.setProjectName(StringUtils.isNotBlank(currentProject.getCnName()) ? currentProject.getCnName() : currentProject.getName());
        newApplication.setRuleGroupName(rules.iterator().next().getRuleGroup().getRuleGroupName());
        newApplication.setRuleDatesource(rules.stream().map(Rule::getRuleDataSources).flatMap(ruleDataSources -> ruleDataSources.stream()).filter(ruleDataSource -> StringUtils.isNotBlank(ruleDataSource.getDbName()) && StringUtils.isNotBlank(ruleDataSource.getTableName())).map(ruleDataSource -> ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + " ").collect(Collectors.joining()));
    }
    applicationDao.saveApplication(newApplication);
    LOGGER.info("Succeed to set application status to [{}], application_id: [{}]", newApplication.getStatus(), newApplication.getId());
}

Example 22 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class RuleBatchServiceImpl method getMultiTemplateRule.

@Override
public List<ExcelMultiTemplateRuleByProject> getMultiTemplateRule(Iterable<Rule> rules, String localeStr) {
    List<ExcelMultiTemplateRuleByProject> lines = new ArrayList<>();
    for (Rule rule : rules) {
        if (!rule.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())) {
            continue;
        }
        ExcelMultiTemplateRuleByProject ruleLinePrefix = new ExcelMultiTemplateRuleByProject();
        // Rule basic info.
        String clusterName = rule.getRuleDataSources().iterator().next().getClusterName();
        basicInfoToExcel(ruleLinePrefix, rule, clusterName);
        List<RuleVariable> filterRuleVariable = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getTemplateMidTableInputMeta().getInputType().equals(TemplateInputTypeEnum.CONDITION.getCode())).collect(Collectors.toList());
        if (filterRuleVariable != null && filterRuleVariable.size() != 0) {
            ruleLinePrefix.setWhereFilter(filterRuleVariable.iterator().next().getValue());
        }
        Boolean sourceAddedFlag = false;
        Boolean targetAddedFlag = false;
        lines.add(ruleLinePrefix);
        ExcelMultiTemplateRuleByProject datasourceExcelRule = new ExcelMultiTemplateRuleByProject(rule.getName());
        for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
            String databaseName = ruleDataSource.getDbName();
            String tableName = ruleDataSource.getTableName();
            String filter = ruleDataSource.getFilter();
            Integer datasourceIndex = ruleDataSource.getDatasourceIndex();
            if (datasourceIndex == 0) {
                datasourceExcelRule.setLeftDbName(databaseName);
                datasourceExcelRule.setLeftTableName(tableName);
                datasourceExcelRule.setLeftFilter(filter);
                datasourceExcelRule.setLeftProxyUser(ruleDataSource.getProxyUser());
                if (ruleDataSource.getLinkisDataSourceId() != null) {
                    datasourceExcelRule.setLeftLinkisDataSourceType(TemplateDataSourceTypeEnum.getMessage(ruleDataSource.getDatasourceType()));
                    datasourceExcelRule.setLeftLinkisDataSourceId(String.valueOf(ruleDataSource.getLinkisDataSourceId()));
                    datasourceExcelRule.setLeftLinkisDataSourceName(ruleDataSource.getLinkisDataSourceName());
                }
                sourceAddedFlag = true;
            } else {
                datasourceExcelRule.setRightDbName(databaseName);
                datasourceExcelRule.setRightTableName(tableName);
                datasourceExcelRule.setRightFilter(filter);
                datasourceExcelRule.setRightProxyUser(ruleDataSource.getProxyUser());
                if (ruleDataSource.getLinkisDataSourceId() != null) {
                    datasourceExcelRule.setRightLlinkisDataSourceType(TemplateDataSourceTypeEnum.getMessage(ruleDataSource.getDatasourceType()));
                    datasourceExcelRule.setRightLinkisDataSourceId(String.valueOf(ruleDataSource.getLinkisDataSourceId()));
                    datasourceExcelRule.setRightLlinkisDataSourceName(ruleDataSource.getLinkisDataSourceName());
                }
                targetAddedFlag = true;
            }
            if (sourceAddedFlag && targetAddedFlag) {
                LOGGER.info("Collect excel line: {}", datasourceExcelRule);
                lines.add(datasourceExcelRule);
            }
        }
        // Get mapping and alarm info to excel.
        getMappingInfoAndAlarm(rule.getRuleDataSourceMappings(), rule.getAlarmConfigs(), lines, ruleLinePrefix, localeStr);
    }
    return lines;
}

Example 23 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class RuleBatchServiceImpl method getTemplateRule.

@Override
public List<ExcelTemplateRuleByProject> getTemplateRule(Iterable<Rule> rules, String localeStr) {
    List<ExcelTemplateRuleByProject> lines = new ArrayList<>();
    for (Rule rule : rules) {
        if (!rule.getRuleType().equals(RuleTypeEnum.SINGLE_TEMPLATE_RULE.getCode())) {
            continue;
        }
        // Rule basic info.
        ExcelTemplateRuleByProject ruleLinePrefix = new ExcelTemplateRuleByProject();
        basicInfoToExcel(ruleLinePrefix, rule);
        lines.add(ruleLinePrefix);
        for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
            String clusterName = ruleDataSource.getClusterName();
            String databaseName = ruleDataSource.getDbName();
            String tableName = ruleDataSource.getTableName();
            String columnName = ruleDataSource.getColName();
            String filter = ruleDataSource.getFilter();
            ExcelTemplateRuleByProject tmp = new ExcelTemplateRuleByProject(rule.getName());
            tmp.setBlackColName(ruleDataSource.getBlackColName());
            tmp.setProxyUser(ruleDataSource.getProxyUser());
            tmp.setFilter(filter);
            tmp.setCluster(clusterName);
            tmp.setDbName(databaseName);
            tmp.setTableName(tableName);
            tmp.setColumnNames(columnName);
            if (ruleDataSource.getLinkisDataSourceId() != null) {
                tmp.setLinkisDataSourceName(ruleDataSource.getLinkisDataSourceName());
                tmp.setLinkisDataSourceId(String.valueOf(ruleDataSource.getLinkisDataSourceId()));
                tmp.setLinkisDataSourceType(TemplateDataSourceTypeEnum.getMessage(ruleDataSource.getDatasourceType()));
            }
            LOGGER.info("Collect excel line: {}", tmp);
            lines.add(tmp);
        }
        for (TemplateMidTableInputMeta templateMidTableInputMeta : rule.getTemplate().getTemplateMidTableInputMetas()) {
            if (TemplateMidTableUtil.shouldResponse(templateMidTableInputMeta)) {
                for (RuleVariable ruleVariable : rule.getRuleVariables()) {
                    if (ruleVariable.getTemplateMidTableInputMeta().equals(templateMidTableInputMeta)) {
                        ExcelTemplateRuleByProject tmp = new ExcelTemplateRuleByProject(rule.getName());
                        String key = templateMidTableInputMeta.getName();
                        String value = StringEscapeUtils.unescapeJava(ruleVariable.getValue());
                        if (templateMidTableInputMeta.getInputType().equals(TemplateInputTypeEnum.REGEXP.getCode())) {
                            if (templateMidTableInputMeta.getRegexpType() != null) {
                                value = ruleVariable.getOriginValue();
                            }
                        }
                        tmp.setArgumentKey(key);
                        tmp.setArgumentValue(value);
                        LOGGER.info("Collect excel line: {}", tmp);
                        lines.add(tmp);
                    }
                }
            }
        }
        alarmConfigsToExcel(rule, lines, localeStr);
    }
    return lines;
}

Example 24 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class RuleDataSourceServiceImpl method checkAndSaveRuleDataSource.

@Override
@Transactional(rollbackFor = { RuntimeException.class, UnExpectedRequestException.class })
public List<RuleDataSource> checkAndSaveRuleDataSource(List<DataSourceRequest> requests, Rule rule, boolean cs, String loginUser) throws UnExpectedRequestException {
    List<RuleDataSource> ruleDataSources = new ArrayList<>();
    for (DataSourceRequest request : requests) {
        RuleDataSource newRuleDataSource = new RuleDataSource();
        // Check Arguments
        DataSourceRequest.checkRequest(request, cs, false);
        if (StringUtils.isNotBlank(request.getLinkisDataSourceType())) {
            newRuleDataSource.setDatasourceType(TemplateDataSourceTypeEnum.getCode(request.getLinkisDataSourceType()));
            newRuleDataSource.setLinkisDataSourceVersionId(request.getLinkisDataSourceVersionId());
            newRuleDataSource.setLinkisDataSourceName(request.getLinkisDataSourceName());
            newRuleDataSource.setLinkisDataSourceId(request.getLinkisDataSourceId());
        }
        newRuleDataSource.setClusterName(request.getClusterName());
        newRuleDataSource.setTableName(request.getTableName());
        newRuleDataSource.setDbName(request.getDbName());
        newRuleDataSource.setFilter(request.getFilter());
        // Saved as: field1: type1, field2: type2
        newRuleDataSource.setBlackColName(request.getBlackList());
        newRuleDataSource.setProxyUser(request.getProxyUser());
        String joinCols = joinColNames(request.getColNames());
        newRuleDataSource.setColName(joinCols);
        newRuleDataSource.setRule(rule);
        if (rule.getTemplate().getTemplateType().equals(RuleTemplateTypeEnum.MULTI_SOURCE_TEMPLATE.getCode())) {
            newRuleDataSource.setDatasourceIndex(request.getDatasourceIndex());
        }
        newRuleDataSource.setProjectId(rule.getProject().getId());
        ruleDataSources.add(newRuleDataSource);
        LOGGER.info("Succeed to save rule_datasource. rule_datasource: {}", newRuleDataSource);
    }
    LOGGER.info("Succeed to save all rule_datasource.");
    return ruleDatasourceDao.saveAllRuleDataSource(ruleDataSources);
}

Example 25 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class RuleDataSourceServiceImpl method updateRuleDataSourceCount.

@Override
public void updateRuleDataSourceCount(Rule ruleInDb, Integer varyAmount) {
    Set<RuleDataSource> ruleDataSources = ruleInDb.getRuleDataSources();
    if (ruleDataSources == null || ruleDataSources.isEmpty()) {
        LOGGER.error("Rule does not have ruledatasource.");
        return;
    }
    for (RuleDataSource ruleDataSource : ruleDataSources) {
        StringBuffer datasourceName = new StringBuffer();
        if (StringUtils.isEmpty(ruleDataSource.getTableName()) || StringUtils.isEmpty(ruleDataSource.getColName())) {
            continue;
        }
        if (StringUtils.isNotBlank(ruleDataSource.getColName())) {
            String[] cols = ruleDataSource.getColName().split(SpecCharEnum.VERTICAL_BAR.getValue());
            for (String col : cols) {
                datasourceName.append(ruleDataSource.getClusterName()).append("-").append(ruleDataSource.getDbName()).append("-").append(ruleDataSource.getTableName()).append("-").append(col);
                Long userId = userDao.findByUsername(ruleInDb.getCreateUser()).getId();
                RuleDataSourceCount ruleDataSourceCount = ruleDataSourceCountRepository.findByDatasourceNameAndUserId(datasourceName.toString(), userId);
                if (ruleDataSourceCount == null) {
                    if (varyAmount > 0) {
                        ruleDataSourceCount = new RuleDataSourceCount(datasourceName.toString(), userId);
                        ruleDataSourceCountRepository.save(ruleDataSourceCount);
                    } else {
                        LOGGER.error("Rule datasource ddes not have related rules.");
                    }
                } else {
                    ruleDataSourceCount.setDatasourceCount(ruleDataSourceCount.getDatasourceCount() + varyAmount);
                    ruleDataSourceCountRepository.save(ruleDataSourceCount);
                    LOGGER.info("Rule [{}] count of datasource is: {}", ruleInDb.getName(), ruleDataSourceCount.getDatasourceCount());
                }
                datasourceName.delete(0, datasourceName.length());
            }
        }
    }
}