Examples with RuleDataSource com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource used on opensource projects

Example 26 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method constructDataSourceRequest.

private List<DataSourceRequest> constructDataSourceRequest(Set<RuleDataSource> ruleDataSources) {
    List<DataSourceRequest> dataSourceRequests = new ArrayList<>(ruleDataSources.size());
    for (RuleDataSource ruleDataSource : ruleDataSources) {
        DataSourceRequest dataSourceRequest = new DataSourceRequest();
        if (ruleDataSource.getLinkisDataSourceId() != null) {
            dataSourceRequest.setLinkisDataSourceId(ruleDataSource.getLinkisDataSourceId());
            dataSourceRequest.setLinkisDataSourceName(ruleDataSource.getLinkisDataSourceName());
            dataSourceRequest.setLinkisDataSourceType(TemplateDataSourceTypeEnum.getMessage(ruleDataSource.getDatasourceType()));
        }
        dataSourceRequest.setClusterName(ruleDataSource.getClusterName());
        dataSourceRequest.setBlackList(ruleDataSource.getBlackColName());
        dataSourceRequest.setProxyUser(ruleDataSource.getProxyUser());
        dataSourceRequest.setTableName(ruleDataSource.getTableName());
        dataSourceRequest.setDbName(ruleDataSource.getDbName());
        dataSourceRequest.setFilter(ruleDataSource.getFilter());
        String colNamesOrigin = ruleDataSource.getColName();
        List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>();
        if (!StringUtils.isBlank(colNamesOrigin)) {
            String[] colNamesSplit = colNamesOrigin.split(SpecCharEnum.VERTICAL_BAR.getValue());
            for (String str : colNamesSplit) {
                DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest();
                dataSourceColumnRequest.setColumnName(str.split(":")[0]);
                dataSourceColumnRequest.setDataType(str.split(":")[1]);
                dataSourceColumnRequests.add(dataSourceColumnRequest);
            }
        }
        dataSourceRequest.setColNames(dataSourceColumnRequests);
        dataSourceRequests.add(dataSourceRequest);
    }
    return dataSourceRequests;
}

Example 27 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method importRuleReal.

private void importRuleReal(RuleNodeRequest ruleNodeRequest, Rule ruleInDb, Rule rule, Project projectInDb, Template template, RuleGroup ruleGroup, Set<AlarmConfig> alarmConfigs, Set<RuleVariable> ruleVariables, Set<RuleDataSource> ruleDataSources, Set<RuleDataSourceMapping> ruleDataSourceMappings) throws IOException, UnExpectedRequestException {
    if (ruleInDb == null) {
        LOGGER.info("Import in first time. That means adding.");
        rule.setProject(projectInDb);
        ruleGroup.setProjectId(projectInDb.getId());
        if (RuleTypeEnum.CUSTOM_RULE.getCode().equals(rule.getRuleType())) {
            Template saveTemplate = ruleTemplateDao.saveTemplate(template);
            Set<Integer> templateDateTypes = ruleDataSources.stream().filter(currRuleDataSource -> currRuleDataSource.getDatasourceType() != null).map(RuleDataSource::getDatasourceType).collect(Collectors.toSet());
            for (Integer templateDataType : templateDateTypes) {
                TemplateDataSourceType templateDataSourceType = new TemplateDataSourceType(templateDataType, saveTemplate);
                templateDataSourceTypeDao.save(templateDataSourceType);
            }
            Set<TemplateStatisticsInputMeta> templateStatisticsInputMetas = new HashSet<>();
            Set<TemplateOutputMeta> templateOutputMetaSet = new HashSet<>();
            if (rule.getOutputName() != null && rule.getFunctionType() != null && rule.getFunctionContent() != null) {
                templateStatisticsInputMetas = templateStatisticsInputMetaService.getAndSaveTemplateStatisticsInputMeta(rule.getOutputName(), rule.getFunctionType(), rule.getFunctionContent(), saveTemplate.getSaveMidTable(), saveTemplate);
                templateOutputMetaSet = templateOutputMetaService.getAndSaveTemplateOutputMeta(rule.getOutputName(), rule.getFunctionType(), saveTemplate.getSaveMidTable(), saveTemplate);
            } else {
                List<RuleMetric> ruleMetrics = alarmConfigs.stream().map(AlarmConfig::getRuleMetric).collect(Collectors.toList());
                for (RuleMetric ruleMetric : ruleMetrics) {
                    templateStatisticsInputMetas.addAll(templateStatisticsInputMetaService.getAndSaveTemplateStatisticsInputMeta(ruleMetric.getName(), FunctionTypeEnum.MAX_FUNCTION.getCode(), ruleMetric.getName(), saveTemplate.getSaveMidTable(), saveTemplate));
                    templateOutputMetaSet.addAll(templateOutputMetaService.getAndSaveTemplateOutputMeta(ruleMetric.getName(), FunctionTypeEnum.MAX_FUNCTION.getCode(), saveTemplate.getSaveMidTable(), saveTemplate));
                }
            }
            saveTemplate.setStatisticAction(templateStatisticsInputMetas);
            saveTemplate.setTemplateOutputMetas(templateOutputMetaSet);
            rule.setTemplate(saveTemplate);
        } else if (RuleTypeEnum.FILE_TEMPLATE_RULE.getCode().equals(rule.getRuleType())) {
            LOGGER.info("Start to import file rule. {}", rule.getName());
            // Save file rule template.
            rule.setTemplate(ruleTemplateDao.saveTemplate(template));
            // Save file rule group
            rule.setRuleGroup(ruleGroup);
            Rule savedRule = ruleDao.saveRule(rule);
            // Save file alarmconfig
            List<AlarmConfig> alarmConfigList = new ArrayList<>();
            for (AlarmConfig alarmConfig : alarmConfigs) {
                ruleMetricSycn(alarmConfig);
                alarmConfig.setRule(savedRule);
                alarmConfigList.add(alarmConfig);
            }
            List<RuleDataSource> ruleDataSourceList = new ArrayList<>();
            for (RuleDataSource ruleDataSource : ruleDataSources) {
                ruleDataSource.setProjectId(projectInDb.getId());
                ruleDataSource.setRule(savedRule);
                ruleDataSourceList.add(ruleDataSource);
            }
            savedRule.setAlarmConfigs(new HashSet<>(alarmConfigDao.saveAllAlarmConfig(alarmConfigList)));
            savedRule.setRuleDataSources(new HashSet<>(ruleDataSourceDao.saveAllRuleDataSource(ruleDataSourceList)));
            LOGGER.info("Finish to import file rule. {}", rule.getName());
            return;
        } else {
            // When trying out the newly created rule template in the development center, the production center needs to synchronize the rule template.
            synchroRuleTemplate(ruleNodeRequest, rule, template, ruleDataSources);
        }
        rule.setRuleGroup(ruleGroup);
        Rule savedRule = ruleDao.saveRule(rule);
        saveRuleInfo(savedRule, template, projectInDb, alarmConfigs, ruleVariables, ruleDataSources, ruleDataSourceMappings);
        if (ruleNodeRequest.getChildRuleObject() != null) {
            importChildRule(savedRule, ruleGroup, ruleNodeRequest, projectInDb);
        }
    } else {
        LOGGER.info("Import multiple times. That is to update.");
        if (RuleTypeEnum.FILE_TEMPLATE_RULE.getCode().equals(rule.getRuleType())) {
            LOGGER.info("Start to update import file rule. {}", rule.getName());
            Set<AlarmConfig> alarmConfigList = new HashSet<>();
            for (AlarmConfig alarmConfig : alarmConfigs) {
                ruleMetricSycn(alarmConfig);
                alarmConfig.setRule(ruleInDb);
                alarmConfigList.add(alarmConfig);
            }
            updateImportedFileRule(rule, ruleInDb, ruleGroup, alarmConfigList, ruleDataSources);
            LOGGER.info("Finish to update import file rule. {}", rule.getName());
            return;
        }
        updateImportedRule(rule, ruleInDb, ruleGroup, ruleNodeRequest, template, alarmConfigs, ruleVariables, ruleDataSources, ruleDataSourceMappings);
    }
}

Example 28 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method importChildRule.

private void importChildRule(Rule parentRule, RuleGroup ruleGroup, RuleNodeRequest ruleNodeRequest, Project projectInDb) throws IOException {
    LOGGER.info("Start to import child rule.");
    ObjectMapper objectMapper = new ObjectMapper();
    Rule childRule = objectMapper.readValue(ruleNodeRequest.getChildRuleObject(), Rule.class);
    Template childTemplate = objectMapper.readValue(ruleNodeRequest.getChildTemplateObject(), Template.class);
    Set<RuleDataSource> childRuleDataSources = objectMapper.readValue(ruleNodeRequest.getChildRuleDataSourcesObject(), new TypeReference<Set<RuleDataSource>>() {
    });
    Set<RuleDataSourceMapping> childRuleDataSourceMappings = objectMapper.readValue(ruleNodeRequest.getChildRuleDataSourceMappingsObject(), new TypeReference<Set<RuleDataSourceMapping>>() {
    });
    Set<AlarmConfig> childAlarmConfigs = objectMapper.readValue(ruleNodeRequest.getChildAlarmConfigsObject(), new TypeReference<Set<AlarmConfig>>() {
    });
    Set<RuleVariable> childRuleVariables = objectMapper.readValue(ruleNodeRequest.getChildRuleVariablesObject(), new TypeReference<Set<RuleVariable>>() {
    });
    childRule.setProject(projectInDb);
    childRule.setTemplate(childTemplate);
    childRule.setRuleGroup(ruleGroup);
    Rule savedRule = ruleDao.saveRule(childRule);
    LOGGER.info("Succeed to save child rule. Rule info is {}", savedRule.toString());
    List<AlarmConfig> alarmConfigList = new ArrayList<>();
    for (AlarmConfig alarmConfig : childAlarmConfigs) {
        alarmConfig.setRule(savedRule);
        alarmConfigList.add(alarmConfig);
    }
    List<RuleVariable> ruleVariablesList = new ArrayList<>();
    for (RuleVariable ruleVariable : childRuleVariables) {
        ruleVariable.setRule(savedRule);
        ruleVariablesList.add(ruleVariable);
    }
    List<RuleDataSource> ruleDataSourceList = new ArrayList<>();
    for (RuleDataSource ruleDataSource : childRuleDataSources) {
        ruleDataSource.setProjectId(projectInDb.getId());
        ruleDataSource.setRule(savedRule);
        ruleDataSourceList.add(ruleDataSource);
    }
    savedRule.setAlarmConfigs(new HashSet<>(alarmConfigDao.saveAllAlarmConfig(alarmConfigList)));
    savedRule.setRuleVariables(new HashSet<>(ruleVariableDao.saveAllRuleVariable(ruleVariablesList)));
    savedRule.setRuleDataSources(new HashSet<>(ruleDataSourceDao.saveAllRuleDataSource(ruleDataSourceList)));
    for (RuleDataSourceMapping ruleDataSourceMapping : childRuleDataSourceMappings) {
        ruleDataSourceMapping.setRule(savedRule);
        ruleDataSourceMappingDao.saveRuleDataSourceMapping(ruleDataSourceMapping);
    }
    savedRule.setParentRule(parentRule);
    parentRule.setChildRule(savedRule);
    LOGGER.info("Succeed to import child rule.");
}

Example 29 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method constructCustomRequest.

private AddCustomRuleRequest constructCustomRequest(Rule rule, RuleGroup ruleGroup) {
    AddCustomRuleRequest addCustomRuleRequest = new AddCustomRuleRequest();
    String newVersion = ruleGroup.getVersion();
    if (StringUtils.isEmpty(newVersion)) {
        String newRuleName = rule.getName() + "_copy_" + ruleGroup.getId();
        LOGGER.info("Custom rule start to be copied. Copied rule name: " + newRuleName);
        addCustomRuleRequest.setRuleName(newRuleName);
        if (StringUtils.isNotBlank(rule.getCnName())) {
            addCustomRuleRequest.setRuleCnName(rule.getCnName() + "_副本");
        }
    } else {
        String oldVersion = rule.getRuleGroup().getVersion();
        LOGGER.info("Custom rule start to be copied. Copied rule name: " + rule.getName() + "_" + ruleGroup.getVersion());
        if (StringUtils.isNotBlank(oldVersion) && rule.getName().endsWith(oldVersion)) {
            addCustomRuleRequest.setRuleName(rule.getName().replace(oldVersion, newVersion));
            if (StringUtils.isNotBlank(rule.getCnName())) {
                addCustomRuleRequest.setRuleCnName(rule.getCnName().replace(oldVersion, newVersion));
            }
        } else {
            addCustomRuleRequest.setRuleName(rule.getName() + "_" + newVersion);
            if (StringUtils.isNotBlank(rule.getCnName())) {
                addCustomRuleRequest.setRuleCnName(rule.getCnName() + "_" + newVersion);
            }
        }
    }
    addCustomRuleRequest.setRuleDetail(rule.getDetail());
    addCustomRuleRequest.setSaveMidTable(rule.getTemplate().getSaveMidTable());
    if (rule.getFunctionType() != null && StringUtils.isNotBlank(rule.getFunctionContent()) && StringUtils.isNotBlank(rule.getFromContent()) && StringUtils.isNotBlank(rule.getWhereContent())) {
        String functionContent = rule.getFunctionContent();
        String whereContent = rule.getWhereContent();
        String fromContent = rule.getFromContent();
        addCustomRuleRequest.setFunctionType(rule.getFunctionType());
        addCustomRuleRequest.setFunctionContent(functionContent);
        addCustomRuleRequest.setWhereContent(whereContent);
        addCustomRuleRequest.setFromContent(fromContent);
    } else {
        addCustomRuleRequest.setSqlCheckArea(rule.getTemplate().getMidTableAction());
    }
    addCustomRuleRequest.setAlarm(true);
    addCustomRuleRequest.setAlarmVariable(constructCustomAlarmConfigRequest(rule.getAlarmConfigs(), addCustomRuleRequest));
    RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
    addCustomRuleRequest.setClusterName(ruleDataSource.getClusterName());
    addCustomRuleRequest.setProxyUser(ruleDataSource.getProxyUser());
    addCustomRuleRequest.setProjectId(ruleGroup.getProjectId());
    addCustomRuleRequest.setRuleGroupId(ruleGroup.getId());
    addCustomRuleRequest.setCsId(rule.getCsId());
    addCustomRuleRequest.setAbortOnFailure(rule.getAbortOnFailure());
    addCustomRuleRequest.setStaticStartupParam(rule.getStaticStartupParam());
    addCustomRuleRequest.setSpecifyStaticStartupParam(rule.getSpecifyStaticStartupParam());
    if (ruleDataSource.getLinkisDataSourceId() != null) {
        addCustomRuleRequest.setLinkisDataSourceId(ruleDataSource.getLinkisDataSourceId());
        addCustomRuleRequest.setLinkisDataSourceName(ruleDataSource.getLinkisDataSourceName());
        addCustomRuleRequest.setLinkisDataSourceType(TemplateDataSourceTypeEnum.getMessage(ruleDataSource.getDatasourceType()));
    }
    return addCustomRuleRequest;
}

Example 30 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class CustomRuleServiceImpl method addCustomRuleReal.

private GeneralResponse<RuleResponse> addCustomRuleReal(AddCustomRuleRequest request, String loginUser) throws UnExpectedRequestException, PermissionDeniedRequestException {
    // Check Arguments
    AddCustomRuleRequest.checkRequest(request);
    String nowDate = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date());
    // Generate Template, TemplateStatisticsInputMeta and save
    Template template = ruleTemplateService.addCustomTemplate(request);
    TemplateDataSourceType templateDataSourceType = new TemplateDataSourceType();
    if (request.getLinkisDataSourceType() == null) {
        templateDataSourceType.setDataSourceTypeId(TemplateDataSourceTypeEnum.HIVE.getCode());
    } else {
        templateDataSourceType.setDataSourceTypeId(TemplateDataSourceTypeEnum.MYSQL.getCode());
    }
    templateDataSourceType.setTemplate(template);
    templateDataSourceTypeDao.save(templateDataSourceType);
    // Save rule, rule_alarm_config and ruleDataSource
    // Check existence of project
    Project projectInDb = projectService.checkProjectExistence(request.getProjectId(), loginUser);
    // Check permissions of project
    List<Integer> permissions = new ArrayList<>();
    permissions.add(ProjectUserPermissionEnum.DEVELOPER.getCode());
    projectService.checkProjectPermission(projectInDb, loginUser, permissions);
    // Check unique of rule name
    ruleService.checkRuleName(request.getRuleName(), projectInDb, null);
    // Check if cluster name is supported
    ruleDataSourceService.checkDataSourceClusterSupport(request.getClusterName());
    RuleGroup ruleGroup;
    if (request.getRuleGroupId() != null) {
        ruleGroup = ruleGroupDao.findById(request.getRuleGroupId());
        if (ruleGroup == null) {
            throw new UnExpectedRequestException(String.format("Rule Group: %s {&CAN_NOT_BE_NULL_OR_EMPTY}", request.getRuleGroupId()));
        }
    } else {
        ruleGroup = ruleGroupDao.saveRuleGroup(new RuleGroup("Group_" + UUID.randomUUID().toString().replace("-", ""), projectInDb.getId()));
    }
    Rule newRule = new Rule();
    // Set basic info.
    setBasicInfo(newRule, projectInDb, ruleGroup, template, loginUser, nowDate, request);
    String csId = request.getCsId();
    // For context service.
    boolean cs = false;
    if (StringUtils.isNotBlank(csId)) {
        newRule.setCsId(csId);
        cs = true;
    }
    String fileId = request.getFileId();
    // For fps file check.
    boolean fps = false;
    if (StringUtils.isNotBlank(fileId)) {
        fps = true;
    }
    String sqlCheckArea = request.getSqlCheckArea();
    // For fps file check.
    boolean sqlCheck = false;
    if (StringUtils.isNotBlank(sqlCheckArea)) {
        sqlCheck = true;
    }
    Rule savedRule = ruleDao.saveRule(newRule);
    LOGGER.info("Succeed to save custom rule, rule_id: {}", savedRule.getId());
    List<AlarmConfig> savedAlarmConfigs = new ArrayList<>();
    if (request.getAlarm()) {
        savedAlarmConfigs = alarmConfigService.checkAndSaveCustomAlarmVariable(request.getAlarmVariable(), savedRule);
        LOGGER.info("Succeed to save alarm_configs, alarm_configs: {}", savedAlarmConfigs);
    }
    List<RuleDataSource> ruleDataSources = ruleDataSourceService.checkAndSaveCustomRuleDataSource(request.getClusterName(), request.getProxyUser(), loginUser, savedRule, cs, sqlCheck, request.getLinkisDataSourceId(), request.getLinkisDataSourceVersionId(), request.getLinkisDataSourceName(), request.getLinkisDataSourceType());
    savedRule.setAlarmConfigs(new HashSet<>(savedAlarmConfigs));
    if (CollectionUtils.isNotEmpty(ruleDataSources)) {
        savedRule.setRuleDataSources(new HashSet<>(ruleDataSources));
    }
    // Update rule count of datasource
    ruleDataSourceService.updateRuleDataSourceCount(savedRule, 1);
    RuleResponse response = new RuleResponse(savedRule);
    LOGGER.info("Succeed to add custom rule, rule_id: {}", savedRule.getId());
    return new GeneralResponse<>("200", "{&SUCCEED_TO_ADD_CUSTOM_RULE}", response);
}