Search in sources :

Example 36 with RuleMetric

use of com.webank.wedatasphere.qualitis.entity.RuleMetric in project Qualitis by WeBankFinTech.

the class SqlTemplateConverter method generateSparkSqlByTask.

/**
 * Convert task into scala code
 * @param rule
 * @param date
 * @param applicationId
 * @param midTableName
 * @param createTime
 * @param partition
 * @param execParams
 * @param count
 * @param runDate
 * @param dataSourceMysqlConnect
 * @return
 * @throws ConvertException
 * @throws RuleVariableNotSupportException
 * @throws RuleVariableNotFoundException
 */
private List<String> generateSparkSqlByTask(Rule rule, Date date, String applicationId, String midTableName, String createTime, StringBuffer partition, Map<String, String> execParams, int count, String runDate, Map<Long, Map> dataSourceMysqlConnect) throws ConvertException, RuleVariableNotSupportException, RuleVariableNotFoundException, UnExpectedRequestException {
    List<String> sqlList = new ArrayList<>();
    // Collect rule metric and build in save sentence sql.
    List<RuleMetric> ruleMetrics = rule.getAlarmConfigs().stream().map(AlarmConfig::getRuleMetric).distinct().collect(Collectors.toList());
    Map<String, Long> ruleMetricMap = new HashMap<>(ruleMetrics.size());
    if (CollectionUtils.isNotEmpty(ruleMetrics)) {
        LOGGER.info("Start to get rule metric for task result save. Rule metrics: {}", Arrays.toString(ruleMetrics.toArray()));
        for (RuleMetric ruleMetric : ruleMetrics) {
            if (ruleMetric != null) {
                ruleMetricMap.put(ruleMetric.getName(), ruleMetric.getId());
            }
        }
        LOGGER.info("Finish to get rule metric for task result save.");
    }
    // Get SQL from template after remove '\n'
    String templateMidTableAction = rule.getTemplate().getMidTableAction().replace("\n", " ");
    Map<String, String> filters = new HashMap<>(2);
    if (CUSTOM_RULE.intValue() == rule.getRuleType()) {
        templateMidTableAction = customMidTableActionUpdate(rule, templateMidTableAction, date, execParams, partition, ruleMetricMap);
    } else if (MUL_SOURCE_RULE.intValue() == rule.getRuleType()) {
        templateMidTableAction = multiMidTableActionUpdate(rule, templateMidTableAction, date, filters);
    }
    // Get input meta from template
    List<RuleVariable> inputMetaRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.TEMPLATE_INPUT_META.getCode())).collect(Collectors.toList());
    // If partition is not specified, replace with filter in rule configuration.
    if (StringUtils.isBlank(partition.toString())) {
        templateMidTableAction = fillPartitionWithRuleConfiguration(partition, rule, templateMidTableAction, inputMetaRuleVariables);
    }
    // Get dbs and tables
    Map<String, String> dbTableMap = new HashMap<>(4);
    // Get mappings
    StringBuffer mappings = new StringBuffer();
    StringBuffer realFilter = new StringBuffer();
    // Get SQL From template and replace all replaceholders
    String midTableAction = replaceVariable(templateMidTableAction, inputMetaRuleVariables, partition.toString(), realFilter, dbTableMap, mappings, date);
    Set<TemplateStatisticsInputMeta> templateStatisticsAction = rule.getTemplate().getStatisticAction();
    Map sourceConnect = new HashMap(8);
    Map targetConnect = new HashMap(8);
    if (dataSourceMysqlConnect != null && dataSourceMysqlConnect.size() > 0) {
        for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
            Map connectParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
            if (connectParams == null) {
                continue;
            }
            if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(0)) {
                // If mysql sec, decrypt password and user name.
                sourceConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
            }
            if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(1)) {
                // If mysql sec, decrypt password and user name.
                targetConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
            }
        }
    }
    sqlList.add("val UUID = java.util.UUID.randomUUID.toString");
    // 跨表规则
    if (RuleTemplateTypeEnum.MULTI_SOURCE_TEMPLATE.getCode().equals(rule.getTemplate().getTemplateType()) && dbTableMap.size() > 0) {
        // Import sql function.
        sqlList.addAll(getImportSql());
        // Generate UUID.
        // Transform original table.
        Set<String> columns = new HashSet<>();
        if (rule.getTemplate().getId().longValue() == MUL_SOURCE_ACCURACY_TEMPLATE_ID.longValue()) {
            // Get accuracy columns.
            columns = rule.getRuleDataSourceMappings().stream().map(RuleDataSourceMapping::getLeftColumnNames).map(column -> column.replace("tmp1.", "").replace("tmp2.", "")).collect(Collectors.toSet());
        }
        if (rule.getTemplate().getId().longValue() == MUL_SOURCE_COMMON_TEMPLATE_ID.longValue()) {
            sqlList.addAll(getCommonTransformSql(dbTableMap, mappings, count, partition.toString(), filters, sourceConnect, targetConnect));
        } else {
            sqlList.addAll(getSpecialTransformSql(dbTableMap, count, partition.toString(), filters, Strings.join(columns, ','), sourceConnect, targetConnect));
            if (optimizationConfig.getLightweightQuery()) {
                count += 3;
            }
        }
        sqlList.addAll(getSaveMidTableSentenceSettings());
        sqlList.addAll(getSaveMidTableSentence(midTableName, count, runDate));
    } else {
        // Generate select statement and save into hive database
        RuleDataSource ruleDataSource = rule.getRuleDataSources().stream().filter(dataSource -> dataSource.getDatasourceIndex() == null).iterator().next();
        Map connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
        if (connParams != null) {
            connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
        }
        sqlList.addAll(generateSparkSqlAndSaveSentence(midTableAction, midTableName, rule.getTemplate(), count, connParams, runDate));
        count++;
    }
    // Generate statistics statement, and save into mysql
    List<RuleVariable> statisticsRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.STATISTICS_ARG.getCode())).collect(Collectors.toList());
    sqlList.addAll(saveStatisticAndSaveMySqlSentence(rule.getId(), ruleMetricMap, templateStatisticsAction, applicationId, statisticsRuleVariables, createTime, count, runDate));
    return sqlList;
}
Also used : RuleMetric(com.webank.wedatasphere.qualitis.entity.RuleMetric) Arrays(java.util.Arrays) RuleTaskDetail(com.webank.wedatasphere.qualitis.bean.RuleTaskDetail) SpecCharEnum(com.webank.wedatasphere.qualitis.constant.SpecCharEnum) Date(java.util.Date) RuleVariable(com.webank.wedatasphere.qualitis.rule.entity.RuleVariable) LoggerFactory(org.slf4j.LoggerFactory) SimpleDateFormat(java.text.SimpleDateFormat) Autowired(org.springframework.beans.factory.annotation.Autowired) Rule(com.webank.wedatasphere.qualitis.rule.entity.Rule) HashMap(java.util.HashMap) StringUtils(org.apache.commons.lang3.StringUtils) ArrayList(java.util.ArrayList) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) DateExprReplaceUtil(com.webank.wedatasphere.qualitis.util.DateExprReplaceUtil) HashSet(java.util.HashSet) AlarmConfig(com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig) Matcher(java.util.regex.Matcher) CollectionUtils(org.apache.commons.collections.CollectionUtils) DataQualityTaskException(com.webank.wedatasphere.qualitis.exception.DataQualityTaskException) Map(java.util.Map) ConvertException(com.webank.wedatasphere.qualitis.exception.ConvertException) RuleVariableNotSupportException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotSupportException) RuleVariableNotFoundException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotFoundException) TemplateMidTableInputMeta(com.webank.wedatasphere.qualitis.rule.entity.TemplateMidTableInputMeta) Logger(org.slf4j.Logger) InputActionStepEnum(com.webank.wedatasphere.qualitis.rule.constant.InputActionStepEnum) AbstractTranslator(com.webank.wedatasphere.qualitis.translator.AbstractTranslator) RuleMetric(com.webank.wedatasphere.qualitis.entity.RuleMetric) Template(com.webank.wedatasphere.qualitis.rule.entity.Template) Set(java.util.Set) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) RuleDataSourceMapping(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSourceMapping) RuleConstraintEnum(com.webank.wedatasphere.qualitis.metadata.constant.RuleConstraintEnum) DataQualityJob(com.webank.wedatasphere.qualitis.bean.DataQualityJob) TemplateInputTypeEnum(com.webank.wedatasphere.qualitis.rule.constant.TemplateInputTypeEnum) List(java.util.List) Component(org.springframework.stereotype.Component) RuleTemplateTypeEnum(com.webank.wedatasphere.qualitis.rule.constant.RuleTemplateTypeEnum) Strings(org.apache.logging.log4j.util.Strings) DataQualityTask(com.webank.wedatasphere.qualitis.bean.DataQualityTask) TemplateStatisticsInputMeta(com.webank.wedatasphere.qualitis.rule.entity.TemplateStatisticsInputMeta) Pattern(java.util.regex.Pattern) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) OptimizationConfig(com.webank.wedatasphere.qualitis.config.OptimizationConfig) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) TemplateStatisticsInputMeta(com.webank.wedatasphere.qualitis.rule.entity.TemplateStatisticsInputMeta) HashMap(java.util.HashMap) Map(java.util.Map) AlarmConfig(com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig) RuleVariable(com.webank.wedatasphere.qualitis.rule.entity.RuleVariable) HashSet(java.util.HashSet)

Aggregations

RuleMetric (com.webank.wedatasphere.qualitis.entity.RuleMetric)36 ArrayList (java.util.ArrayList)23 UnExpectedRequestException (com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException)19 AlarmConfig (com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig)15 RuleMetricDepartmentUser (com.webank.wedatasphere.qualitis.entity.RuleMetricDepartmentUser)10 User (com.webank.wedatasphere.qualitis.entity.User)10 Department (com.webank.wedatasphere.qualitis.entity.Department)9 UserRole (com.webank.wedatasphere.qualitis.entity.UserRole)9 List (java.util.List)9 ExcelRuleMetric (com.webank.wedatasphere.qualitis.project.excel.ExcelRuleMetric)8 GeneralResponse (com.webank.wedatasphere.qualitis.response.GeneralResponse)8 Rule (com.webank.wedatasphere.qualitis.rule.entity.Rule)7 TemplateOutputMeta (com.webank.wedatasphere.qualitis.rule.entity.TemplateOutputMeta)7 CustomAlarmConfigRequest (com.webank.wedatasphere.qualitis.rule.request.CustomAlarmConfigRequest)7 FileAlarmConfigRequest (com.webank.wedatasphere.qualitis.rule.request.FileAlarmConfigRequest)7 PermissionDeniedRequestException (com.webank.wedatasphere.qualitis.exception.PermissionDeniedRequestException)6 RuleMetricResponse (com.webank.wedatasphere.qualitis.response.RuleMetricResponse)6 RuleDataSource (com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource)5 Date (java.util.Date)5 Template (com.webank.wedatasphere.qualitis.rule.entity.Template)4