Examples with AlarmConfig com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig used on opensource projects

Example 11 with AlarmConfig

use of com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method customReSaveDateSource.

private void customReSaveDateSource(Rule currentRule, Map<String, String> execParams, String clusterName, Date date) throws UnExpectedRequestException, SemanticException, ParseException {
    String midTableAction = currentRule.getTemplate().getMidTableAction();
    if (StringUtils.isNotBlank(currentRule.getWhereContent())) {
        midTableAction = midTableAction.replace("${filter}", currentRule.getWhereContent());
    }
    // Replace placeholder with key and value for sql parse check.
    for (String key : execParams.keySet()) {
        midTableAction = midTableAction.replace("${" + key + "}", execParams.get(key));
    }
    midTableAction = DateExprReplaceUtil.replaceRunDate(date, midTableAction);
    // Save datasource in first execution(Only possible data sources related to fps).
    boolean firstExecution = CollectionUtils.isEmpty(currentRule.getRuleDataSources().stream().filter(ruleDataSource -> !ORIGINAL_INDEX.equals(ruleDataSource.getDatasourceIndex())).collect(Collectors.toSet()));
    if (!firstExecution) {
        return;
    }
    RuleDataSource originalRuleDataSource = currentRule.getRuleDataSources().stream().filter(ruleDataSource -> ORIGINAL_INDEX.equals(ruleDataSource.getDatasourceIndex())).iterator().next();
    if (StringUtils.isEmpty(clusterName)) {
        clusterName = originalRuleDataSource.getClusterName();
    }
    Map<String, List<String>> dbAndTables = new HashMap<>(2);
    // 0.16.0 for mysql oracle, hive parse, use druid.
    LOGGER.info("Parse sql: " + midTableAction);
    List<RuleMetric> ruleMetrics = currentRule.getAlarmConfigs().stream().map(AlarmConfig::getRuleMetric).distinct().collect(Collectors.toList());
    Set<String> ruleMetricNames = ruleMetrics.stream().map(RuleMetric::getName).collect(Collectors.toSet());
    for (String ruleMetricName : ruleMetricNames) {
        String cleanRuleMetricName = ruleMetricName.replace("-", "_");
        midTableAction = midTableAction.replace(ruleMetricName, cleanRuleMetricName);
    }
    LOGGER.info("Parse sql after replace rule metric names: " + midTableAction);
    if (originalRuleDataSource.getLinkisDataSourceId() != null) {
        DbType dbType = JdbcConstants.MYSQL;
        MySqlSchemaStatVisitor visitor = new MySqlSchemaStatVisitor();
        List<SQLStatement> stmtList = SQLUtils.parseStatements(midTableAction, dbType);
        for (int i = 0; i < stmtList.size(); i++) {
            SQLStatement stmt = stmtList.get(i);
            stmt.accept(visitor);
            Map<Name, TableStat> tableStatMap = visitor.getTables();
            dbAndTables = toDbAndTables(dbAndTables, tableStatMap);
        }
    } else {
        HiveSqlParser hiveSqlParser = new HiveSqlParser();
        dbAndTables = hiveSqlParser.checkSelectSqlAndGetDbAndTable(midTableAction);
    }
    LOGGER.info("Db and tables: " + dbAndTables.toString());
    for (String db : dbAndTables.keySet()) {
        List<RuleDataSource> ruleDataSources = new ArrayList<>();
        for (String table : dbAndTables.get(db)) {
            RuleDataSource ruleDataSource = new RuleDataSource();
            ruleDataSource.setClusterName(clusterName);
            ruleDataSource.setProjectId(currentRule.getProject().getId());
            ruleDataSource.setDbName(db.toLowerCase());
            ruleDataSource.setRule(currentRule);
            ruleDataSource.setTableName(table);
            ruleDataSource.setProxyUser(originalRuleDataSource.getProxyUser());
            ruleDataSource.setLinkisDataSourceId(originalRuleDataSource.getLinkisDataSourceId());
            ruleDataSource.setLinkisDataSourceName(originalRuleDataSource.getLinkisDataSourceName());
            ruleDataSource.setLinkisDataSourceVersionId(originalRuleDataSource.getLinkisDataSourceVersionId());
            ruleDataSources.add(ruleDataSource);
        }
        Set<RuleDataSource> ruleDataSourcesExist = currentRule.getRuleDataSources();
        ruleDataSourcesExist.addAll(ruleDataSourceDao.saveAllRuleDataSource(ruleDataSources).stream().collect(Collectors.toSet()));
        currentRule.setRuleDataSources(ruleDataSourcesExist);
        ruleDao.saveRule(currentRule);
    }
}

Example 12 with AlarmConfig

use of com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig in project Qualitis by WeBankFinTech.

the class RuleBatchServiceImpl method getMappingInfoAndAlarm.

private void getMappingInfoAndAlarm(Set<RuleDataSourceMapping> ruleDataSourceMappings, Set<AlarmConfig> alarmConfigs, List<ExcelMultiTemplateRuleByProject> lines, ExcelMultiTemplateRuleByProject ruleLinePrefix, String localeStr) {
    for (RuleDataSourceMapping mapping : ruleDataSourceMappings) {
        ExcelMultiTemplateRuleByProject tmp = new ExcelMultiTemplateRuleByProject(ruleLinePrefix.getRuleName());
        tmp.setLeftMappingStatement(mapping.getLeftStatement());
        tmp.setLeftMappingNames(mapping.getLeftColumnNames());
        tmp.setLeftMappingTypes(mapping.getLeftColumnTypes());
        tmp.setMappingOperation(MappingOperationEnum.getByCode(mapping.getOperation()).getSymbol());
        tmp.setRightMappingStatement(mapping.getRightStatement());
        tmp.setRightMappingNames(mapping.getRightColumnNames());
        tmp.setRightMappingTypes(mapping.getRightColumnTypes());
        LOGGER.info("Collect excel line: {}", tmp);
        lines.add(tmp);
    }
    for (AlarmConfig alarmConfig : alarmConfigs) {
        Double threshold = alarmConfig.getThreshold();
        String alarmOutputName = alarmConfig.getTemplateOutputMeta().getOutputName();
        String alarmCompareType = CompareTypeEnum.getCompareTypeName(alarmConfig.getCompareType());
        String checkTemplateName = CheckTemplateEnum.getCheckTemplateName(alarmConfig.getCheckTemplate(), localeStr);
        ExcelMultiTemplateRuleByProject tmp = new ExcelMultiTemplateRuleByProject(ruleLinePrefix.getRuleName());
        tmp.setCompareType(alarmCompareType);
        tmp.setAlarmCheckName(alarmOutputName);
        tmp.setCheckTemplateName(checkTemplateName);
        tmp.setThreshold(String.valueOf(threshold));
        RuleMetric ruleMetric = alarmConfig.getRuleMetric();
        // Recod rule metric info (unique code).
        if (ruleMetric != null) {
            String enCode = ruleMetric.getEnCode();
            tmp.setRuleMetricEnCode(enCode);
            tmp.setRuleMetricName(ruleMetric.getName());
        }
        tmp.setUploadRuleMetricValue(alarmConfig.getUploadRuleMetricValue());
        tmp.setUploadAbnormalValue(alarmConfig.getUploadAbnormalValue());
        LOGGER.info("Collect excel line: {}", tmp);
        lines.add(tmp);
    }
}

Example 13 with AlarmConfig

use of com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig in project Qualitis by WeBankFinTech.

the class RuleBatchServiceImpl method getCustomRule.

@Override
public List<ExcelCustomRuleByProject> getCustomRule(Iterable<Rule> rules, String localeStr) {
    List<ExcelCustomRuleByProject> lines = new ArrayList<>();
    for (Rule rule : rules) {
        if (!rule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
            continue;
        }
        ExcelCustomRuleByProject ruleLinePrefix = new ExcelCustomRuleByProject();
        basicInfoToExcel(ruleLinePrefix, rule);
        lines.add(ruleLinePrefix);
        for (AlarmConfig alarmConfig : rule.getAlarmConfigs()) {
            Double threshold = alarmConfig.getThreshold();
            String alarmOutputName = alarmConfig.getTemplateOutputMeta().getOutputName();
            String alarmCompareType = CompareTypeEnum.getCompareTypeName(alarmConfig.getCompareType());
            String checkTemplateName = CheckTemplateEnum.getCheckTemplateName(alarmConfig.getCheckTemplate(), localeStr);
            ExcelCustomRuleByProject tmp = new ExcelCustomRuleByProject(rule.getName());
            tmp.setCompareType(alarmCompareType);
            tmp.setAlarmCheckName(alarmOutputName);
            tmp.setThreshold(String.valueOf(threshold));
            tmp.setCheckTemplateName(checkTemplateName);
            RuleMetric ruleMetric = alarmConfig.getRuleMetric();
            // Recod rule metric info (unique code).
            if (ruleMetric != null) {
                String enCode = ruleMetric.getEnCode();
                tmp.setRuleMetricEnCode(enCode);
                tmp.setRuleMetricName(ruleMetric.getName());
            }
            tmp.setUploadRuleMetricValue(alarmConfig.getUploadRuleMetricValue());
            tmp.setUploadAbnormalValue(alarmConfig.getUploadAbnormalValue());
            lines.add(tmp);
        }
    }
    return lines;
}

Example 14 with AlarmConfig

use of com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method importRuleReal.

private void importRuleReal(RuleNodeRequest ruleNodeRequest, Rule ruleInDb, Rule rule, Project projectInDb, Template template, RuleGroup ruleGroup, Set<AlarmConfig> alarmConfigs, Set<RuleVariable> ruleVariables, Set<RuleDataSource> ruleDataSources, Set<RuleDataSourceMapping> ruleDataSourceMappings) throws IOException, UnExpectedRequestException {
    if (ruleInDb == null) {
        LOGGER.info("Import in first time. That means adding.");
        rule.setProject(projectInDb);
        ruleGroup.setProjectId(projectInDb.getId());
        if (RuleTypeEnum.CUSTOM_RULE.getCode().equals(rule.getRuleType())) {
            Template saveTemplate = ruleTemplateDao.saveTemplate(template);
            Set<Integer> templateDateTypes = ruleDataSources.stream().filter(currRuleDataSource -> currRuleDataSource.getDatasourceType() != null).map(RuleDataSource::getDatasourceType).collect(Collectors.toSet());
            for (Integer templateDataType : templateDateTypes) {
                TemplateDataSourceType templateDataSourceType = new TemplateDataSourceType(templateDataType, saveTemplate);
                templateDataSourceTypeDao.save(templateDataSourceType);
            }
            Set<TemplateStatisticsInputMeta> templateStatisticsInputMetas = new HashSet<>();
            Set<TemplateOutputMeta> templateOutputMetaSet = new HashSet<>();
            if (rule.getOutputName() != null && rule.getFunctionType() != null && rule.getFunctionContent() != null) {
                templateStatisticsInputMetas = templateStatisticsInputMetaService.getAndSaveTemplateStatisticsInputMeta(rule.getOutputName(), rule.getFunctionType(), rule.getFunctionContent(), saveTemplate.getSaveMidTable(), saveTemplate);
                templateOutputMetaSet = templateOutputMetaService.getAndSaveTemplateOutputMeta(rule.getOutputName(), rule.getFunctionType(), saveTemplate.getSaveMidTable(), saveTemplate);
            } else {
                List<RuleMetric> ruleMetrics = alarmConfigs.stream().map(AlarmConfig::getRuleMetric).collect(Collectors.toList());
                for (RuleMetric ruleMetric : ruleMetrics) {
                    templateStatisticsInputMetas.addAll(templateStatisticsInputMetaService.getAndSaveTemplateStatisticsInputMeta(ruleMetric.getName(), FunctionTypeEnum.MAX_FUNCTION.getCode(), ruleMetric.getName(), saveTemplate.getSaveMidTable(), saveTemplate));
                    templateOutputMetaSet.addAll(templateOutputMetaService.getAndSaveTemplateOutputMeta(ruleMetric.getName(), FunctionTypeEnum.MAX_FUNCTION.getCode(), saveTemplate.getSaveMidTable(), saveTemplate));
                }
            }
            saveTemplate.setStatisticAction(templateStatisticsInputMetas);
            saveTemplate.setTemplateOutputMetas(templateOutputMetaSet);
            rule.setTemplate(saveTemplate);
        } else if (RuleTypeEnum.FILE_TEMPLATE_RULE.getCode().equals(rule.getRuleType())) {
            LOGGER.info("Start to import file rule. {}", rule.getName());
            // Save file rule template.
            rule.setTemplate(ruleTemplateDao.saveTemplate(template));
            // Save file rule group
            rule.setRuleGroup(ruleGroup);
            Rule savedRule = ruleDao.saveRule(rule);
            // Save file alarmconfig
            List<AlarmConfig> alarmConfigList = new ArrayList<>();
            for (AlarmConfig alarmConfig : alarmConfigs) {
                ruleMetricSycn(alarmConfig);
                alarmConfig.setRule(savedRule);
                alarmConfigList.add(alarmConfig);
            }
            List<RuleDataSource> ruleDataSourceList = new ArrayList<>();
            for (RuleDataSource ruleDataSource : ruleDataSources) {
                ruleDataSource.setProjectId(projectInDb.getId());
                ruleDataSource.setRule(savedRule);
                ruleDataSourceList.add(ruleDataSource);
            }
            savedRule.setAlarmConfigs(new HashSet<>(alarmConfigDao.saveAllAlarmConfig(alarmConfigList)));
            savedRule.setRuleDataSources(new HashSet<>(ruleDataSourceDao.saveAllRuleDataSource(ruleDataSourceList)));
            LOGGER.info("Finish to import file rule. {}", rule.getName());
            return;
        } else {
            // When trying out the newly created rule template in the development center, the production center needs to synchronize the rule template.
            synchroRuleTemplate(ruleNodeRequest, rule, template, ruleDataSources);
        }
        rule.setRuleGroup(ruleGroup);
        Rule savedRule = ruleDao.saveRule(rule);
        saveRuleInfo(savedRule, template, projectInDb, alarmConfigs, ruleVariables, ruleDataSources, ruleDataSourceMappings);
        if (ruleNodeRequest.getChildRuleObject() != null) {
            importChildRule(savedRule, ruleGroup, ruleNodeRequest, projectInDb);
        }
    } else {
        LOGGER.info("Import multiple times. That is to update.");
        if (RuleTypeEnum.FILE_TEMPLATE_RULE.getCode().equals(rule.getRuleType())) {
            LOGGER.info("Start to update import file rule. {}", rule.getName());
            Set<AlarmConfig> alarmConfigList = new HashSet<>();
            for (AlarmConfig alarmConfig : alarmConfigs) {
                ruleMetricSycn(alarmConfig);
                alarmConfig.setRule(ruleInDb);
                alarmConfigList.add(alarmConfig);
            }
            updateImportedFileRule(rule, ruleInDb, ruleGroup, alarmConfigList, ruleDataSources);
            LOGGER.info("Finish to update import file rule. {}", rule.getName());
            return;
        }
        updateImportedRule(rule, ruleInDb, ruleGroup, ruleNodeRequest, template, alarmConfigs, ruleVariables, ruleDataSources, ruleDataSourceMappings);
    }
}

Example 15 with AlarmConfig

use of com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method constructAlarmConfigRequest.

private List<AlarmConfigRequest> constructAlarmConfigRequest(Set<AlarmConfig> alarmConfigs) {
    List<AlarmConfigRequest> alarmConfigRequests = new ArrayList<>(alarmConfigs.size());
    for (AlarmConfig alarmConfig : alarmConfigs) {
        AlarmConfigRequest alarmConfigRequest = new AlarmConfigRequest();
        alarmConfigRequest.setOutputMetaId(alarmConfig.getTemplateOutputMeta().getId());
        alarmConfigRequest.setCheckTemplate(alarmConfig.getCheckTemplate());
        alarmConfigRequest.setCompareType(alarmConfig.getCompareType());
        alarmConfigRequest.setThreshold(alarmConfig.getThreshold());
        RuleMetric ruleMetric = alarmConfig.getRuleMetric();
        alarmConfigRequest.setRuleMetricEnCode(ruleMetric != null ? ruleMetric.getEnCode() : "");
        alarmConfigRequest.setUploadAbnormalValue(alarmConfig.getUploadAbnormalValue());
        alarmConfigRequest.setUploadRuleMetricValue(alarmConfig.getUploadRuleMetricValue());
        alarmConfigRequests.add(alarmConfigRequest);
    }
    return alarmConfigRequests;
}