Examples with RuleVariable com.webank.wedatasphere.qualitis.rule.entity.RuleVariable used on opensource projects

Example 6 with RuleVariable

use of com.webank.wedatasphere.qualitis.rule.entity.RuleVariable in project Qualitis by WeBankFinTech.

the class RuleBatchServiceImpl method getTemplateRule.

@Override
public List<ExcelTemplateRuleByProject> getTemplateRule(Iterable<Rule> rules, String localeStr) {
    List<ExcelTemplateRuleByProject> lines = new ArrayList<>();
    for (Rule rule : rules) {
        if (!rule.getRuleType().equals(RuleTypeEnum.SINGLE_TEMPLATE_RULE.getCode())) {
            continue;
        }
        // Rule basic info.
        ExcelTemplateRuleByProject ruleLinePrefix = new ExcelTemplateRuleByProject();
        basicInfoToExcel(ruleLinePrefix, rule);
        lines.add(ruleLinePrefix);
        for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
            String clusterName = ruleDataSource.getClusterName();
            String databaseName = ruleDataSource.getDbName();
            String tableName = ruleDataSource.getTableName();
            String columnName = ruleDataSource.getColName();
            String filter = ruleDataSource.getFilter();
            ExcelTemplateRuleByProject tmp = new ExcelTemplateRuleByProject(rule.getName());
            tmp.setBlackColName(ruleDataSource.getBlackColName());
            tmp.setProxyUser(ruleDataSource.getProxyUser());
            tmp.setFilter(filter);
            tmp.setCluster(clusterName);
            tmp.setDbName(databaseName);
            tmp.setTableName(tableName);
            tmp.setColumnNames(columnName);
            if (ruleDataSource.getLinkisDataSourceId() != null) {
                tmp.setLinkisDataSourceName(ruleDataSource.getLinkisDataSourceName());
                tmp.setLinkisDataSourceId(String.valueOf(ruleDataSource.getLinkisDataSourceId()));
                tmp.setLinkisDataSourceType(TemplateDataSourceTypeEnum.getMessage(ruleDataSource.getDatasourceType()));
            }
            LOGGER.info("Collect excel line: {}", tmp);
            lines.add(tmp);
        }
        for (TemplateMidTableInputMeta templateMidTableInputMeta : rule.getTemplate().getTemplateMidTableInputMetas()) {
            if (TemplateMidTableUtil.shouldResponse(templateMidTableInputMeta)) {
                for (RuleVariable ruleVariable : rule.getRuleVariables()) {
                    if (ruleVariable.getTemplateMidTableInputMeta().equals(templateMidTableInputMeta)) {
                        ExcelTemplateRuleByProject tmp = new ExcelTemplateRuleByProject(rule.getName());
                        String key = templateMidTableInputMeta.getName();
                        String value = StringEscapeUtils.unescapeJava(ruleVariable.getValue());
                        if (templateMidTableInputMeta.getInputType().equals(TemplateInputTypeEnum.REGEXP.getCode())) {
                            if (templateMidTableInputMeta.getRegexpType() != null) {
                                value = ruleVariable.getOriginValue();
                            }
                        }
                        tmp.setArgumentKey(key);
                        tmp.setArgumentValue(value);
                        LOGGER.info("Collect excel line: {}", tmp);
                        lines.add(tmp);
                    }
                }
            }
        }
        alarmConfigsToExcel(rule, lines, localeStr);
    }
    return lines;
}

Example 7 with RuleVariable

use of com.webank.wedatasphere.qualitis.rule.entity.RuleVariable in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method constructTemplateArgumentRequest.

private List<TemplateArgumentRequest> constructTemplateArgumentRequest(Rule rule) {
    List<TemplateArgumentRequest> templateArgumentRequests = new ArrayList<>(rule.getTemplate().getTemplateMidTableInputMetas().size());
    for (TemplateMidTableInputMeta templateMidTableInputMeta : rule.getTemplate().getTemplateMidTableInputMetas()) {
        if (TemplateMidTableUtil.shouldResponse(templateMidTableInputMeta)) {
            for (RuleVariable ruleVariable : rule.getRuleVariables()) {
                TemplateArgumentRequest templateArgumentRequest = new TemplateArgumentRequest();
                if (ruleVariable.getTemplateMidTableInputMeta().equals(templateMidTableInputMeta)) {
                    String value = StringEscapeUtils.unescapeJava(ruleVariable.getValue());
                    if (templateMidTableInputMeta.getInputType().equals(TemplateInputTypeEnum.REGEXP.getCode())) {
                        if (templateMidTableInputMeta.getRegexpType() != null) {
                            value = ruleVariable.getOriginValue();
                        }
                    }
                    templateArgumentRequest.setArgumentStep(InputActionStepEnum.TEMPLATE_INPUT_META.getCode());
                    templateArgumentRequest.setArgumentId(templateMidTableInputMeta.getId());
                    templateArgumentRequest.setArgumentValue(value);
                    templateArgumentRequests.add(templateArgumentRequest);
                }
            }
        }
    }
    return templateArgumentRequests;
}

Example 8 with RuleVariable

use of com.webank.wedatasphere.qualitis.rule.entity.RuleVariable in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method importChildRule.

private void importChildRule(Rule parentRule, RuleGroup ruleGroup, RuleNodeRequest ruleNodeRequest, Project projectInDb) throws IOException {
    LOGGER.info("Start to import child rule.");
    ObjectMapper objectMapper = new ObjectMapper();
    Rule childRule = objectMapper.readValue(ruleNodeRequest.getChildRuleObject(), Rule.class);
    Template childTemplate = objectMapper.readValue(ruleNodeRequest.getChildTemplateObject(), Template.class);
    Set<RuleDataSource> childRuleDataSources = objectMapper.readValue(ruleNodeRequest.getChildRuleDataSourcesObject(), new TypeReference<Set<RuleDataSource>>() {
    });
    Set<RuleDataSourceMapping> childRuleDataSourceMappings = objectMapper.readValue(ruleNodeRequest.getChildRuleDataSourceMappingsObject(), new TypeReference<Set<RuleDataSourceMapping>>() {
    });
    Set<AlarmConfig> childAlarmConfigs = objectMapper.readValue(ruleNodeRequest.getChildAlarmConfigsObject(), new TypeReference<Set<AlarmConfig>>() {
    });
    Set<RuleVariable> childRuleVariables = objectMapper.readValue(ruleNodeRequest.getChildRuleVariablesObject(), new TypeReference<Set<RuleVariable>>() {
    });
    childRule.setProject(projectInDb);
    childRule.setTemplate(childTemplate);
    childRule.setRuleGroup(ruleGroup);
    Rule savedRule = ruleDao.saveRule(childRule);
    LOGGER.info("Succeed to save child rule. Rule info is {}", savedRule.toString());
    List<AlarmConfig> alarmConfigList = new ArrayList<>();
    for (AlarmConfig alarmConfig : childAlarmConfigs) {
        alarmConfig.setRule(savedRule);
        alarmConfigList.add(alarmConfig);
    }
    List<RuleVariable> ruleVariablesList = new ArrayList<>();
    for (RuleVariable ruleVariable : childRuleVariables) {
        ruleVariable.setRule(savedRule);
        ruleVariablesList.add(ruleVariable);
    }
    List<RuleDataSource> ruleDataSourceList = new ArrayList<>();
    for (RuleDataSource ruleDataSource : childRuleDataSources) {
        ruleDataSource.setProjectId(projectInDb.getId());
        ruleDataSource.setRule(savedRule);
        ruleDataSourceList.add(ruleDataSource);
    }
    savedRule.setAlarmConfigs(new HashSet<>(alarmConfigDao.saveAllAlarmConfig(alarmConfigList)));
    savedRule.setRuleVariables(new HashSet<>(ruleVariableDao.saveAllRuleVariable(ruleVariablesList)));
    savedRule.setRuleDataSources(new HashSet<>(ruleDataSourceDao.saveAllRuleDataSource(ruleDataSourceList)));
    for (RuleDataSourceMapping ruleDataSourceMapping : childRuleDataSourceMappings) {
        ruleDataSourceMapping.setRule(savedRule);
        ruleDataSourceMappingDao.saveRuleDataSourceMapping(ruleDataSourceMapping);
    }
    savedRule.setParentRule(parentRule);
    parentRule.setChildRule(savedRule);
    LOGGER.info("Succeed to import child rule.");
}

Example 9 with RuleVariable

use of com.webank.wedatasphere.qualitis.rule.entity.RuleVariable in project Qualitis by WeBankFinTech.

the class SqlTemplateConverter method generateSparkSqlByTask.

/**
 * Convert task into scala code
 * @param rule
 * @param date
 * @param applicationId
 * @param midTableName
 * @param createTime
 * @param partition
 * @param execParams
 * @param count
 * @param runDate
 * @param dataSourceMysqlConnect
 * @return
 * @throws ConvertException
 * @throws RuleVariableNotSupportException
 * @throws RuleVariableNotFoundException
 */
private List<String> generateSparkSqlByTask(Rule rule, Date date, String applicationId, String midTableName, String createTime, StringBuffer partition, Map<String, String> execParams, int count, String runDate, Map<Long, Map> dataSourceMysqlConnect) throws ConvertException, RuleVariableNotSupportException, RuleVariableNotFoundException, UnExpectedRequestException {
    List<String> sqlList = new ArrayList<>();
    // Collect rule metric and build in save sentence sql.
    List<RuleMetric> ruleMetrics = rule.getAlarmConfigs().stream().map(AlarmConfig::getRuleMetric).distinct().collect(Collectors.toList());
    Map<String, Long> ruleMetricMap = new HashMap<>(ruleMetrics.size());
    if (CollectionUtils.isNotEmpty(ruleMetrics)) {
        LOGGER.info("Start to get rule metric for task result save. Rule metrics: {}", Arrays.toString(ruleMetrics.toArray()));
        for (RuleMetric ruleMetric : ruleMetrics) {
            if (ruleMetric != null) {
                ruleMetricMap.put(ruleMetric.getName(), ruleMetric.getId());
            }
        }
        LOGGER.info("Finish to get rule metric for task result save.");
    }
    // Get SQL from template after remove '\n'
    String templateMidTableAction = rule.getTemplate().getMidTableAction().replace("\n", " ");
    Map<String, String> filters = new HashMap<>(2);
    if (CUSTOM_RULE.intValue() == rule.getRuleType()) {
        templateMidTableAction = customMidTableActionUpdate(rule, templateMidTableAction, date, execParams, partition, ruleMetricMap);
    } else if (MUL_SOURCE_RULE.intValue() == rule.getRuleType()) {
        templateMidTableAction = multiMidTableActionUpdate(rule, templateMidTableAction, date, filters);
    }
    // Get input meta from template
    List<RuleVariable> inputMetaRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.TEMPLATE_INPUT_META.getCode())).collect(Collectors.toList());
    // If partition is not specified, replace with filter in rule configuration.
    if (StringUtils.isBlank(partition.toString())) {
        templateMidTableAction = fillPartitionWithRuleConfiguration(partition, rule, templateMidTableAction, inputMetaRuleVariables);
    }
    // Get dbs and tables
    Map<String, String> dbTableMap = new HashMap<>(4);
    // Get mappings
    StringBuffer mappings = new StringBuffer();
    StringBuffer realFilter = new StringBuffer();
    // Get SQL From template and replace all replaceholders
    String midTableAction = replaceVariable(templateMidTableAction, inputMetaRuleVariables, partition.toString(), realFilter, dbTableMap, mappings, date);
    Set<TemplateStatisticsInputMeta> templateStatisticsAction = rule.getTemplate().getStatisticAction();
    Map sourceConnect = new HashMap(8);
    Map targetConnect = new HashMap(8);
    if (dataSourceMysqlConnect != null && dataSourceMysqlConnect.size() > 0) {
        for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
            Map connectParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
            if (connectParams == null) {
                continue;
            }
            if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(0)) {
                // If mysql sec, decrypt password and user name.
                sourceConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
            }
            if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(1)) {
                // If mysql sec, decrypt password and user name.
                targetConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
            }
        }
    }
    sqlList.add("val UUID = java.util.UUID.randomUUID.toString");
    // 跨表规则
    if (RuleTemplateTypeEnum.MULTI_SOURCE_TEMPLATE.getCode().equals(rule.getTemplate().getTemplateType()) && dbTableMap.size() > 0) {
        // Import sql function.
        sqlList.addAll(getImportSql());
        // Generate UUID.
        // Transform original table.
        Set<String> columns = new HashSet<>();
        if (rule.getTemplate().getId().longValue() == MUL_SOURCE_ACCURACY_TEMPLATE_ID.longValue()) {
            // Get accuracy columns.
            columns = rule.getRuleDataSourceMappings().stream().map(RuleDataSourceMapping::getLeftColumnNames).map(column -> column.replace("tmp1.", "").replace("tmp2.", "")).collect(Collectors.toSet());
        }
        if (rule.getTemplate().getId().longValue() == MUL_SOURCE_COMMON_TEMPLATE_ID.longValue()) {
            sqlList.addAll(getCommonTransformSql(dbTableMap, mappings, count, partition.toString(), filters, sourceConnect, targetConnect));
        } else {
            sqlList.addAll(getSpecialTransformSql(dbTableMap, count, partition.toString(), filters, Strings.join(columns, ','), sourceConnect, targetConnect));
            if (optimizationConfig.getLightweightQuery()) {
                count += 3;
            }
        }
        sqlList.addAll(getSaveMidTableSentenceSettings());
        sqlList.addAll(getSaveMidTableSentence(midTableName, count, runDate));
    } else {
        // Generate select statement and save into hive database
        RuleDataSource ruleDataSource = rule.getRuleDataSources().stream().filter(dataSource -> dataSource.getDatasourceIndex() == null).iterator().next();
        Map connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
        if (connParams != null) {
            connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
        }
        sqlList.addAll(generateSparkSqlAndSaveSentence(midTableAction, midTableName, rule.getTemplate(), count, connParams, runDate));
        count++;
    }
    // Generate statistics statement, and save into mysql
    List<RuleVariable> statisticsRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.STATISTICS_ARG.getCode())).collect(Collectors.toList());
    sqlList.addAll(saveStatisticAndSaveMySqlSentence(rule.getId(), ruleMetricMap, templateStatisticsAction, applicationId, statisticsRuleVariables, createTime, count, runDate));
    return sqlList;
}

Example 10 with RuleVariable

use of com.webank.wedatasphere.qualitis.rule.entity.RuleVariable in project Qualitis by WeBankFinTech.

the class SqlTemplateConverter method replaceVariable.

/**
 * Replace all placeholder of template sql
 * @param template
 * @param variables
 * @param filter
 * @param realFilter
 * @param dbTableMap for pick up source db.table & target db.table
 * @param mappings
 * @param date
 * @return
 * @throws ConvertException
 */
private String replaceVariable(String template, List<RuleVariable> variables, String filter, StringBuffer realFilter, Map<String, String> dbTableMap, StringBuffer mappings, Date date) throws ConvertException, UnExpectedRequestException {
    String sqlAction = template;
    if (StringUtils.isNotBlank(filter)) {
        String tmpfilter = DateExprReplaceUtil.replaceFilter(date, filter);
        sqlAction = sqlAction.replace(FILTER_PLACEHOLDER, tmpfilter);
        realFilter.append(tmpfilter);
        LOGGER.info("Succeed to replace {} into {}", FILTER_PLACEHOLDER, tmpfilter);
    } else {
        realFilter.append("true");
    }
    for (RuleVariable ruleVariable : variables) {
        String midInputMetaPlaceHolder = ruleVariable.getTemplateMidTableInputMeta().getPlaceholder();
        String placeHolder = "\\$\\{" + midInputMetaPlaceHolder + "}";
        // GeT source db and table, target db and table.
        if ("source_db".equals(midInputMetaPlaceHolder)) {
            if (StringUtils.isNotBlank(ruleVariable.getValue())) {
                dbTableMap.put("source_db", ruleVariable.getValue() + ".");
            } else {
                dbTableMap.put("source_db", "");
            }
        } else if ("source_table".equals(midInputMetaPlaceHolder)) {
            dbTableMap.put("source_table", ruleVariable.getValue());
        } else if ("target_table".equals(midInputMetaPlaceHolder)) {
            dbTableMap.put("target_table", ruleVariable.getValue());
        } else if ("target_db".equals(midInputMetaPlaceHolder)) {
            if (StringUtils.isNotBlank(ruleVariable.getValue())) {
                dbTableMap.put("target_db", ruleVariable.getValue() + ".");
            } else {
                dbTableMap.put("target_db", "");
            }
        } else if ("mapping_argument".equals(midInputMetaPlaceHolder)) {
            mappings.append(ruleVariable.getValue());
        }
        // Fix issue of wedget node in the front.
        if ("\\$\\{field}".equals(placeHolder)) {
            Matcher matcher = AGGREGATE_FUNC_PATTERN.matcher(ruleVariable.getValue());
            while (matcher.find()) {
                String[] funcs = matcher.group().split("\n");
                for (String func : funcs) {
                    ruleVariable.setValue(ruleVariable.getValue().replace(func, "`" + func + "`"));
                }
            }
        }
        // Fix replacement issue that db is null when running workflow.
        if (ruleVariable.getValue() == null || "".equals(ruleVariable.getValue())) {
            sqlAction = sqlAction.replaceAll(placeHolder + ".", "");
        } else {
            sqlAction = sqlAction.replaceAll(placeHolder, ruleVariable.getValue());
        }
        LOGGER.info("Succeed to replace {} into {}", placeHolder, ruleVariable.getValue());
    }
    if (PLACEHOLDER_PATTERN.matcher(sqlAction).matches()) {
        throw new ConvertException("Unable to convert SQL, replacing placeholders failed, still having placeholder.");
    }
    return sqlAction;
}