Examples with Rule com.webank.wedatasphere.qualitis.rule.entity.Rule used on opensource projects

Example 56 with Rule

use of com.webank.wedatasphere.qualitis.rule.entity.Rule in project Qualitis by WeBankFinTech.

the class FileRuleServiceImpl method modifyRuleDetailReal.

private GeneralResponse<RuleResponse> modifyRuleDetailReal(ModifyFileRuleRequest request, String loginUser) throws UnExpectedRequestException, PermissionDeniedRequestException {
    // Check Arguments
    ModifyFileRuleRequest.checkRequest(request);
    // Check existence of rule
    Rule ruleInDb = ruleDao.findById(request.getRuleId());
    if (ruleInDb == null) {
        throw new UnExpectedRequestException("rule_id [" + request.getRuleId() + "] {&DOES_NOT_EXIST}");
    }
    // Check existence of project
    Project projectInDb = projectService.checkProjectExistence(ruleInDb.getProject().getId(), loginUser);
    // Check permissions of project
    List<Integer> permissions = new ArrayList<>();
    permissions.add(ProjectUserPermissionEnum.DEVELOPER.getCode());
    projectService.checkProjectPermission(projectInDb, loginUser, permissions);
    if (!ruleInDb.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())) {
        throw new UnExpectedRequestException("rule(id: [" + request.getRuleId() + "]) {&IS_NOT_A_FILE_RULE}");
    }
    LOGGER.info("Succeed to find rule. rule_id: {}", ruleInDb.getId());
    String nowDate = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date());
    String csId = request.getCsId();
    boolean cs = false;
    if (StringUtils.isNotBlank(csId)) {
        cs = true;
        ruleInDb.setCsId(csId);
    } else {
        LOGGER.info("Delete context service ID. rule_id: {}， cs_id: {}", ruleInDb.getId(), ruleInDb.getCsId());
        ruleInDb.setCsId(csId);
        LOGGER.info("After delete context service ID. rule_id: {}， cs_id: {}", ruleInDb.getId(), ruleInDb.getCsId());
    }
    // Check cluster support
    ruleDataSourceService.checkDataSourceClusterSupport(request.getDatasource().getClusterName());
    // Delete alarm config by file rule
    alarmConfigService.deleteByRule(ruleInDb);
    LOGGER.info("Succeed to delete all alarm_config. rule_id: {}", ruleInDb.getId());
    // Delete template of file rule
    ruleTemplateService.deleteFileRuleTemplate(ruleInDb.getTemplate().getId());
    LOGGER.info("Succeed to delete file rule template. rule_id: {}", request.getRuleId());
    // Delete rule datasource of file rule
    ruleDataSourceService.deleteByRule(ruleInDb);
    LOGGER.info("Succeed to delete all rule_dataSources. rule_id: {}", ruleInDb.getId());
    // Update rule count of datasource
    ruleDataSourceService.updateRuleDataSourceCount(ruleInDb, -1);
    // Check existence of rule
    AddFileRuleRequest addFileRuleRequest = new AddFileRuleRequest();
    BeanUtils.copyProperties(request, addFileRuleRequest);
    Template template = ruleTemplateService.addFileTemplate(addFileRuleRequest);
    // Save rule
    ruleInDb.setRuleType(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode());
    ruleInDb.setTemplate(template);
    ruleInDb.setName(request.getRuleName());
    ruleInDb.setCnName(request.getRuleCnName());
    ruleInDb.setDetail(request.getRuleDetail());
    ruleInDb.setAlarm(request.getAlarm());
    ruleInDb.setRuleTemplateName(template.getName());
    ruleInDb.setAbortOnFailure(request.getAbortOnFailure());
    ruleInDb.setModifyUser(loginUser);
    ruleInDb.setModifyTime(nowDate);
    Rule savedRule = ruleDao.saveRule(ruleInDb);
    LOGGER.info("Succeed to save file rule, rule_id: {}", savedRule.getId());
    List<AlarmConfig> savedAlarmConfigs = new ArrayList<>();
    if (request.getAlarm()) {
        savedAlarmConfigs = alarmConfigService.checkAndSaveFileAlarmVariable(request.getAlarmVariable(), savedRule);
        LOGGER.info("Succeed to save alarm_configs, alarm_configs: {}", savedAlarmConfigs);
    }
    List<RuleDataSource> ruleDataSources = new ArrayList<>();
    ruleDataSources.add(ruleDataSourceService.checkAndSaveFileRuleDataSource(request.getDatasource(), savedRule, cs));
    savedRule.setAlarmConfigs(new HashSet<>(savedAlarmConfigs));
    savedRule.setRuleDataSources(new HashSet<>(ruleDataSources));
    // Update rule count of datasource
    ruleDataSourceService.updateRuleDataSourceCount(ruleInDb, 1);
    RuleResponse response = new RuleResponse(savedRule);
    LOGGER.info("Succeed to modify file rule, rule_id: {}", savedRule.getId());
    // projectEventService.record(savedRule.getProject().getId(), loginUser, "modify", "file rule[name= " + savedRule.getName() + "].", EventTypeEnum.MODIFY_PROJECT.getCode());
    return new GeneralResponse<>("200", "{&SUCCEED_TO_MODIFY_FILE_RULE}", response);
}

Example 57 with Rule

use of com.webank.wedatasphere.qualitis.rule.entity.Rule in project Qualitis by WeBankFinTech.

the class RuleGroupController method getRuleByRuleGroupId.

@GET
@Path("/{rule_group_id}")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public GeneralResponse<?> getRuleByRuleGroupId(@PathParam("rule_group_id") Long ruleGroupId) throws UnExpectedRequestException {
    try {
        // 查看ruleGroup是否存在
        RuleGroup ruleGroupInDb = ruleGroupDao.findById(ruleGroupId);
        if (ruleGroupInDb == null) {
            throw new UnExpectedRequestException(String.format("Rule Group: %s {&DOES_NOT_EXIST}", ruleGroupId));
        }
        List<RuleResponse> ruleList = ruleDao.findByRuleGroup(ruleGroupInDb).stream().map(rule -> new RuleResponse(rule)).collect(Collectors.toList());
        return new GeneralResponse<>("200", "Succeed to find rules by rule group id", new RuleGroupResponse(ruleGroupId, ruleList));
    } catch (UnExpectedRequestException e) {
        throw new UnExpectedRequestException(e.getMessage());
    } catch (Exception e) {
        LOGGER.error("Failed to get rules by rule group id. rule_group_id: {}, caused by: {}", ruleGroupId, e.getMessage(), e);
        return new GeneralResponse<>("500", "{&FAILED_TO_GET_RULES_BY_RULE_GROUP}", null);
    }
}

Example 58 with Rule

use of com.webank.wedatasphere.qualitis.rule.entity.Rule in project Qualitis by WeBankFinTech.

the class SqlTemplateConverter method generateSparkSqlByTask.

/**
 * Convert task into scala code
 * @param rule
 * @param date
 * @param applicationId
 * @param midTableName
 * @param createTime
 * @param partition
 * @param execParams
 * @param count
 * @param runDate
 * @param dataSourceMysqlConnect
 * @return
 * @throws ConvertException
 * @throws RuleVariableNotSupportException
 * @throws RuleVariableNotFoundException
 */
private List<String> generateSparkSqlByTask(Rule rule, Date date, String applicationId, String midTableName, String createTime, StringBuffer partition, Map<String, String> execParams, int count, String runDate, Map<Long, Map> dataSourceMysqlConnect) throws ConvertException, RuleVariableNotSupportException, RuleVariableNotFoundException, UnExpectedRequestException {
    List<String> sqlList = new ArrayList<>();
    // Collect rule metric and build in save sentence sql.
    List<RuleMetric> ruleMetrics = rule.getAlarmConfigs().stream().map(AlarmConfig::getRuleMetric).distinct().collect(Collectors.toList());
    Map<String, Long> ruleMetricMap = new HashMap<>(ruleMetrics.size());
    if (CollectionUtils.isNotEmpty(ruleMetrics)) {
        LOGGER.info("Start to get rule metric for task result save. Rule metrics: {}", Arrays.toString(ruleMetrics.toArray()));
        for (RuleMetric ruleMetric : ruleMetrics) {
            if (ruleMetric != null) {
                ruleMetricMap.put(ruleMetric.getName(), ruleMetric.getId());
            }
        }
        LOGGER.info("Finish to get rule metric for task result save.");
    }
    // Get SQL from template after remove '\n'
    String templateMidTableAction = rule.getTemplate().getMidTableAction().replace("\n", " ");
    Map<String, String> filters = new HashMap<>(2);
    if (CUSTOM_RULE.intValue() == rule.getRuleType()) {
        templateMidTableAction = customMidTableActionUpdate(rule, templateMidTableAction, date, execParams, partition, ruleMetricMap);
    } else if (MUL_SOURCE_RULE.intValue() == rule.getRuleType()) {
        templateMidTableAction = multiMidTableActionUpdate(rule, templateMidTableAction, date, filters);
    }
    // Get input meta from template
    List<RuleVariable> inputMetaRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.TEMPLATE_INPUT_META.getCode())).collect(Collectors.toList());
    // If partition is not specified, replace with filter in rule configuration.
    if (StringUtils.isBlank(partition.toString())) {
        templateMidTableAction = fillPartitionWithRuleConfiguration(partition, rule, templateMidTableAction, inputMetaRuleVariables);
    }
    // Get dbs and tables
    Map<String, String> dbTableMap = new HashMap<>(4);
    // Get mappings
    StringBuffer mappings = new StringBuffer();
    StringBuffer realFilter = new StringBuffer();
    // Get SQL From template and replace all replaceholders
    String midTableAction = replaceVariable(templateMidTableAction, inputMetaRuleVariables, partition.toString(), realFilter, dbTableMap, mappings, date);
    Set<TemplateStatisticsInputMeta> templateStatisticsAction = rule.getTemplate().getStatisticAction();
    Map sourceConnect = new HashMap(8);
    Map targetConnect = new HashMap(8);
    if (dataSourceMysqlConnect != null && dataSourceMysqlConnect.size() > 0) {
        for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
            Map connectParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
            if (connectParams == null) {
                continue;
            }
            if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(0)) {
                // If mysql sec, decrypt password and user name.
                sourceConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
            }
            if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(1)) {
                // If mysql sec, decrypt password and user name.
                targetConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
            }
        }
    }
    sqlList.add("val UUID = java.util.UUID.randomUUID.toString");
    // 跨表规则
    if (RuleTemplateTypeEnum.MULTI_SOURCE_TEMPLATE.getCode().equals(rule.getTemplate().getTemplateType()) && dbTableMap.size() > 0) {
        // Import sql function.
        sqlList.addAll(getImportSql());
        // Generate UUID.
        // Transform original table.
        Set<String> columns = new HashSet<>();
        if (rule.getTemplate().getId().longValue() == MUL_SOURCE_ACCURACY_TEMPLATE_ID.longValue()) {
            // Get accuracy columns.
            columns = rule.getRuleDataSourceMappings().stream().map(RuleDataSourceMapping::getLeftColumnNames).map(column -> column.replace("tmp1.", "").replace("tmp2.", "")).collect(Collectors.toSet());
        }
        if (rule.getTemplate().getId().longValue() == MUL_SOURCE_COMMON_TEMPLATE_ID.longValue()) {
            sqlList.addAll(getCommonTransformSql(dbTableMap, mappings, count, partition.toString(), filters, sourceConnect, targetConnect));
        } else {
            sqlList.addAll(getSpecialTransformSql(dbTableMap, count, partition.toString(), filters, Strings.join(columns, ','), sourceConnect, targetConnect));
            if (optimizationConfig.getLightweightQuery()) {
                count += 3;
            }
        }
        sqlList.addAll(getSaveMidTableSentenceSettings());
        sqlList.addAll(getSaveMidTableSentence(midTableName, count, runDate));
    } else {
        // Generate select statement and save into hive database
        RuleDataSource ruleDataSource = rule.getRuleDataSources().stream().filter(dataSource -> dataSource.getDatasourceIndex() == null).iterator().next();
        Map connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
        if (connParams != null) {
            connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
        }
        sqlList.addAll(generateSparkSqlAndSaveSentence(midTableAction, midTableName, rule.getTemplate(), count, connParams, runDate));
        count++;
    }
    // Generate statistics statement, and save into mysql
    List<RuleVariable> statisticsRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.STATISTICS_ARG.getCode())).collect(Collectors.toList());
    sqlList.addAll(saveStatisticAndSaveMySqlSentence(rule.getId(), ruleMetricMap, templateStatisticsAction, applicationId, statisticsRuleVariables, createTime, count, runDate));
    return sqlList;
}

Example 59 with Rule

use of com.webank.wedatasphere.qualitis.rule.entity.Rule in project Qualitis by WeBankFinTech.

the class SameDataSourceTaskDivider method divide.

@Override
public List<DataQualityTask> divide(List<Rule> rules, String applicationId, String createTime, String partition, Date date, String database, String user, Integer threshold) throws ArgumentException {
    LOGGER.info("Start to classify rules by datasource");
    Map<String, List<Rule>> sameDataSourceRule = new HashMap<>(4);
    for (Rule rule : rules) {
        String key = getKey(rule, user);
        // Rules without specific execution parameters can be split into the same task, and rules with execution parameters must be treated as a separate task.
        Boolean specifyStaticStartupParam = (rule.getSpecifyStaticStartupParam() != null && rule.getSpecifyStaticStartupParam());
        if (sameDataSourceRule.containsKey(key) && !specifyStaticStartupParam) {
            sameDataSourceRule.get(key).add(rule);
        } else if (specifyStaticStartupParam) {
            List<Rule> tmp = new ArrayList<>();
            tmp.add(rule);
            sameDataSourceRule.put(UUID.randomUUID().toString().replace("-", "") + "." + key, tmp);
        } else {
            List<Rule> tmp = new ArrayList<>();
            tmp.add(rule);
            sameDataSourceRule.put(key, tmp);
        }
    }
    LOGGER.info("Succeed to classify rules by datasource. Result: {}", sameDataSourceRule);
    List<DataQualityTask> result = new ArrayList<>();
    for (String key : sameDataSourceRule.keySet()) {
        List<Rule> ruleList = sameDataSourceRule.get(key);
        String ruleStartup = ruleList.stream().map(Rule::getStaticStartupParam).filter(staticStartupParam -> StringUtils.isNotBlank(staticStartupParam)).collect(Collectors.joining());
        List<Long> ruleIdList = ruleList.stream().map(Rule::getId).collect(Collectors.toList());
        LOGGER.info("Start to divide rules: {} into a task.", ruleIdList);
        LOGGER.info("Start to divide rules. Key: {}", key);
        String[] keys = key.split("\\.");
        String proxyUser = keys[keys.length - 1];
        List<RuleTaskDetail> ruleTaskDetails = new ArrayList<>();
        if (StringUtils.isNotBlank(proxyUser) && database.contains("_ind")) {
            database = proxyUser.concat("_ind");
        }
        for (Rule rule : ruleList) {
            String tableName = generateTable(rule);
            String midTableName = database + "." + tableName;
            if (ruleTaskDetails.size() < threshold) {
                ruleTaskDetails.add(new RuleTaskDetail(rule, midTableName));
            } else {
                List<RuleTaskDetail> ruleTaskDetailCopy = new ArrayList<>();
                ruleTaskDetailCopy.addAll(ruleTaskDetails);
                DataQualityTask tmp = new DataQualityTask(applicationId, createTime, partition, ruleTaskDetailCopy);
                if (StringUtils.isNotBlank(ruleStartup)) {
                    tmp.setStartupParam(ruleStartup);
                }
                if (StringUtils.isNotBlank(proxyUser)) {
                    LOGGER.info("Start to divide rules. Proxy user: {}", proxyUser);
                    tmp.setUser(proxyUser);
                }
                result.add(tmp);
                ruleTaskDetails = new ArrayList<>();
            }
        }
        if (ruleTaskDetails.size() > 0) {
            DataQualityTask tmp = new DataQualityTask(applicationId, createTime, partition, ruleTaskDetails);
            if (StringUtils.isNotBlank(ruleStartup)) {
                tmp.setStartupParam(ruleStartup);
            }
            if (StringUtils.isNotBlank(proxyUser)) {
                tmp.setUser(proxyUser);
            }
            result.add(tmp);
            LOGGER.info("Succeed to divide rules: {} into a task {}", ruleIdList, tmp);
        }
    }
    LOGGER.info("Succeed to divide all rules into tasks. result: {}", result);
    return result;
}

Example 60 with Rule

use of com.webank.wedatasphere.qualitis.rule.entity.Rule in project Qualitis by WeBankFinTech.

the class SameDataSourceTaskDivider method getKey.

private String getKey(Rule rule, String user) throws ArgumentException {
    if (rule.getRuleDataSources().size() != 0) {
        List<RuleDataSource> ruleDataSourceList = rule.getRuleDataSources().stream().filter(dataSource -> StringUtils.isNotBlank(dataSource.getDbName())).collect(Collectors.toList());
        RuleDataSource ruleDataSource;
        if (CollectionUtils.isNotEmpty(ruleDataSourceList)) {
            ruleDataSource = ruleDataSourceList.iterator().next();
        } else {
            ruleDataSource = rule.getRuleDataSources().iterator().next();
        }
        String proxyUser = ruleDataSource.getProxyUser();
        if (StringUtils.isNotBlank(proxyUser)) {
            return ruleDataSource.getClusterName() + "." + ruleDataSource.getDbName() + "." + proxyUser;
        }
        return ruleDataSource.getClusterName() + "." + ruleDataSource.getDbName() + "." + user;
    }
    throw new ArgumentException("Error! Rule variables miss data");
}