use of com.webank.wedatasphere.qualitis.rule.entity.Template in project Qualitis by WeBankFinTech.
the class CustomRuleServiceImpl method modifyCustomRuleReal.
private GeneralResponse<RuleResponse> modifyCustomRuleReal(ModifyCustomRuleRequest request, String loginUser) throws UnExpectedRequestException, MetaDataAcquireFailedException, PermissionDeniedRequestException {
ModifyCustomRuleRequest.checkRequest(request);
Rule ruleInDb = ruleDao.findById(request.getRuleId());
if (ruleInDb == null) {
throw new UnExpectedRequestException("rule_id [" + request.getRuleId() + "] {&DOES_NOT_EXIST}");
}
Project projectInDb = projectService.checkProjectExistence(ruleInDb.getProject().getId(), loginUser);
if (!ruleInDb.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
throw new UnExpectedRequestException("rule_id: [" + request.getRuleId() + "]) {&IS_NOT_A_CUSTOM_RULE}");
}
// Check permissions of project
List<Integer> permissions = new ArrayList<>();
permissions.add(ProjectUserPermissionEnum.DEVELOPER.getCode());
projectService.checkProjectPermission(projectInDb, loginUser, permissions);
LOGGER.info("Succeed to find custom rule. rule_id: {}", ruleInDb.getId());
// Check existence of project name
ruleService.checkRuleName(request.getRuleName(), ruleInDb.getProject(), ruleInDb.getId());
// Check if cluster name supported
ruleDataSourceService.checkDataSourceClusterSupport(request.getClusterName());
String nowDate = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date());
// Delete alarm config by custom rule
alarmConfigService.deleteByRule(ruleInDb);
LOGGER.info("Succeed to delete all alarm_config. rule_id: {}", ruleInDb.getId());
// Delete template of custom rule
ruleTemplateService.deleteCustomTemplate(ruleInDb.getTemplate());
LOGGER.info("Succeed to delete custom rule template. rule_id: {}", request.getRuleId());
// Delete rule datasource of custom rule
ruleDataSourceService.deleteByRule(ruleInDb);
LOGGER.info("Succeed to delete all rule_dataSources. rule_id: {}", ruleInDb.getId());
// Update rule count of datasource
ruleDataSourceService.updateRuleDataSourceCount(ruleInDb, -1);
// Save template, alarm config, rule datasource of custom rule
AddCustomRuleRequest addCustomRuleRequest = new AddCustomRuleRequest();
BeanUtils.copyProperties(request, addCustomRuleRequest);
Template template = ruleTemplateService.addCustomTemplate(addCustomRuleRequest);
TemplateDataSourceType templateDataSourceType = new TemplateDataSourceType();
if (request.getLinkisDataSourceType() == null) {
templateDataSourceType.setDataSourceTypeId(TemplateDataSourceTypeEnum.HIVE.getCode());
} else {
templateDataSourceType.setDataSourceTypeId(TemplateDataSourceTypeEnum.MYSQL.getCode());
}
templateDataSourceType.setTemplate(template);
templateDataSourceTypeDao.save(templateDataSourceType);
// Modify custom rule and save
setBasicInfo(ruleInDb, template, loginUser, nowDate, request);
String csId = request.getCsId();
ruleInDb.setCsId(csId);
boolean cs = false;
if (StringUtils.isNotBlank(csId)) {
cs = true;
}
String fileId = request.getFileId();
boolean fps = false;
if (StringUtils.isNotBlank(fileId)) {
fps = true;
}
String sqlCheckArea = request.getSqlCheckArea();
boolean sqlCheck = false;
if (StringUtils.isNotBlank(sqlCheckArea)) {
sqlCheck = true;
}
Rule savedRule = ruleDao.saveRule(ruleInDb);
// Save alarm config and rule datasource
List<AlarmConfig> savedAlarmConfigs = new ArrayList<>();
if (request.getAlarm()) {
savedAlarmConfigs = alarmConfigService.checkAndSaveCustomAlarmVariable(request.getAlarmVariable(), savedRule);
LOGGER.info("Succeed to save alarm_configs, alarm_configs: {}", savedAlarmConfigs);
}
List<RuleDataSource> ruleDataSources = ruleDataSourceService.checkAndSaveCustomRuleDataSource(request.getClusterName(), request.getProxyUser(), loginUser, savedRule, cs, sqlCheck, request.getLinkisDataSourceId(), request.getLinkisDataSourceVersionId(), request.getLinkisDataSourceName(), request.getLinkisDataSourceType());
savedRule.setAlarmConfigs(new HashSet<>(savedAlarmConfigs));
if (CollectionUtils.isNotEmpty(ruleDataSources)) {
savedRule.setRuleDataSources(new HashSet<>(ruleDataSources));
}
// Update rule count of datasource
ruleDataSourceService.updateRuleDataSourceCount(ruleInDb, 1);
RuleResponse response = new RuleResponse(savedRule);
LOGGER.info("Succeed to modify custom rule, rule_id: {}", savedRule.getId());
// projectEventService.record(savedRule.getProject().getId(), loginUser, "modify", "custom rule[name= " + savedRule.getName() + "].", EventTypeEnum.MODIFY_PROJECT.getCode());
return new GeneralResponse<>("200", "{&SUCCEED_TO_MODIFY_CUSTOM_RULE}", response);
}
use of com.webank.wedatasphere.qualitis.rule.entity.Template in project Qualitis by WeBankFinTech.
the class FileRuleServiceImpl method modifyRuleDetailReal.
private GeneralResponse<RuleResponse> modifyRuleDetailReal(ModifyFileRuleRequest request, String loginUser) throws UnExpectedRequestException, PermissionDeniedRequestException {
// Check Arguments
ModifyFileRuleRequest.checkRequest(request);
// Check existence of rule
Rule ruleInDb = ruleDao.findById(request.getRuleId());
if (ruleInDb == null) {
throw new UnExpectedRequestException("rule_id [" + request.getRuleId() + "] {&DOES_NOT_EXIST}");
}
// Check existence of project
Project projectInDb = projectService.checkProjectExistence(ruleInDb.getProject().getId(), loginUser);
// Check permissions of project
List<Integer> permissions = new ArrayList<>();
permissions.add(ProjectUserPermissionEnum.DEVELOPER.getCode());
projectService.checkProjectPermission(projectInDb, loginUser, permissions);
if (!ruleInDb.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())) {
throw new UnExpectedRequestException("rule(id: [" + request.getRuleId() + "]) {&IS_NOT_A_FILE_RULE}");
}
LOGGER.info("Succeed to find rule. rule_id: {}", ruleInDb.getId());
String nowDate = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date());
String csId = request.getCsId();
boolean cs = false;
if (StringUtils.isNotBlank(csId)) {
cs = true;
ruleInDb.setCsId(csId);
} else {
LOGGER.info("Delete context service ID. rule_id: {}, cs_id: {}", ruleInDb.getId(), ruleInDb.getCsId());
ruleInDb.setCsId(csId);
LOGGER.info("After delete context service ID. rule_id: {}, cs_id: {}", ruleInDb.getId(), ruleInDb.getCsId());
}
// Check cluster support
ruleDataSourceService.checkDataSourceClusterSupport(request.getDatasource().getClusterName());
// Delete alarm config by file rule
alarmConfigService.deleteByRule(ruleInDb);
LOGGER.info("Succeed to delete all alarm_config. rule_id: {}", ruleInDb.getId());
// Delete template of file rule
ruleTemplateService.deleteFileRuleTemplate(ruleInDb.getTemplate().getId());
LOGGER.info("Succeed to delete file rule template. rule_id: {}", request.getRuleId());
// Delete rule datasource of file rule
ruleDataSourceService.deleteByRule(ruleInDb);
LOGGER.info("Succeed to delete all rule_dataSources. rule_id: {}", ruleInDb.getId());
// Update rule count of datasource
ruleDataSourceService.updateRuleDataSourceCount(ruleInDb, -1);
// Check existence of rule
AddFileRuleRequest addFileRuleRequest = new AddFileRuleRequest();
BeanUtils.copyProperties(request, addFileRuleRequest);
Template template = ruleTemplateService.addFileTemplate(addFileRuleRequest);
// Save rule
ruleInDb.setRuleType(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode());
ruleInDb.setTemplate(template);
ruleInDb.setName(request.getRuleName());
ruleInDb.setCnName(request.getRuleCnName());
ruleInDb.setDetail(request.getRuleDetail());
ruleInDb.setAlarm(request.getAlarm());
ruleInDb.setRuleTemplateName(template.getName());
ruleInDb.setAbortOnFailure(request.getAbortOnFailure());
ruleInDb.setModifyUser(loginUser);
ruleInDb.setModifyTime(nowDate);
Rule savedRule = ruleDao.saveRule(ruleInDb);
LOGGER.info("Succeed to save file rule, rule_id: {}", savedRule.getId());
List<AlarmConfig> savedAlarmConfigs = new ArrayList<>();
if (request.getAlarm()) {
savedAlarmConfigs = alarmConfigService.checkAndSaveFileAlarmVariable(request.getAlarmVariable(), savedRule);
LOGGER.info("Succeed to save alarm_configs, alarm_configs: {}", savedAlarmConfigs);
}
List<RuleDataSource> ruleDataSources = new ArrayList<>();
ruleDataSources.add(ruleDataSourceService.checkAndSaveFileRuleDataSource(request.getDatasource(), savedRule, cs));
savedRule.setAlarmConfigs(new HashSet<>(savedAlarmConfigs));
savedRule.setRuleDataSources(new HashSet<>(ruleDataSources));
// Update rule count of datasource
ruleDataSourceService.updateRuleDataSourceCount(ruleInDb, 1);
RuleResponse response = new RuleResponse(savedRule);
LOGGER.info("Succeed to modify file rule, rule_id: {}", savedRule.getId());
// projectEventService.record(savedRule.getProject().getId(), loginUser, "modify", "file rule[name= " + savedRule.getName() + "].", EventTypeEnum.MODIFY_PROJECT.getCode());
return new GeneralResponse<>("200", "{&SUCCEED_TO_MODIFY_FILE_RULE}", response);
}
use of com.webank.wedatasphere.qualitis.rule.entity.Template in project Qualitis by WeBankFinTech.
the class RuleTemplateServiceImpl method getRuleTemplateMeta.
@Override
public GeneralResponse<TemplateMetaResponse> getRuleTemplateMeta(Long ruleTemplateId) throws UnExpectedRequestException {
// Find rule template by id
Template templateInDb = ruleTemplateDao.findById(ruleTemplateId);
if (null == templateInDb) {
throw new UnExpectedRequestException("rule_template_id {&CAN_NOT_BE_NULL_OR_EMPTY}");
}
// Find input meta data by template
List<TemplateMidTableInputMeta> templateMidTableInputMetas = templateMidTableInputMetaDao.findByRuleTemplate(templateInDb);
List<TemplateOutputMeta> templateOutputMetas = templateOutputMetaDao.findByRuleTemplate(templateInDb);
List<Integer> types = templateDataSourceTypeDao.findByTemplate(templateInDb).stream().map(TemplateDataSourceType::getDataSourceTypeId).collect(Collectors.toList());
TemplateMetaResponse response = new TemplateMetaResponse(templateInDb, templateMidTableInputMetas, templateOutputMetas, types);
LOGGER.info("Succeed to get rule_template. rule_template_id: {}", ruleTemplateId);
return new GeneralResponse<>("200", "{&GET_RULE_TEMPLATE_META_SUCCESSFULLY}", response);
}
use of com.webank.wedatasphere.qualitis.rule.entity.Template in project Qualitis by WeBankFinTech.
the class SqlTemplateConverter method generateSparkSqlByTask.
/**
* Convert task into scala code
* @param rule
* @param date
* @param applicationId
* @param midTableName
* @param createTime
* @param partition
* @param execParams
* @param count
* @param runDate
* @param dataSourceMysqlConnect
* @return
* @throws ConvertException
* @throws RuleVariableNotSupportException
* @throws RuleVariableNotFoundException
*/
private List<String> generateSparkSqlByTask(Rule rule, Date date, String applicationId, String midTableName, String createTime, StringBuffer partition, Map<String, String> execParams, int count, String runDate, Map<Long, Map> dataSourceMysqlConnect) throws ConvertException, RuleVariableNotSupportException, RuleVariableNotFoundException, UnExpectedRequestException {
List<String> sqlList = new ArrayList<>();
// Collect rule metric and build in save sentence sql.
List<RuleMetric> ruleMetrics = rule.getAlarmConfigs().stream().map(AlarmConfig::getRuleMetric).distinct().collect(Collectors.toList());
Map<String, Long> ruleMetricMap = new HashMap<>(ruleMetrics.size());
if (CollectionUtils.isNotEmpty(ruleMetrics)) {
LOGGER.info("Start to get rule metric for task result save. Rule metrics: {}", Arrays.toString(ruleMetrics.toArray()));
for (RuleMetric ruleMetric : ruleMetrics) {
if (ruleMetric != null) {
ruleMetricMap.put(ruleMetric.getName(), ruleMetric.getId());
}
}
LOGGER.info("Finish to get rule metric for task result save.");
}
// Get SQL from template after remove '\n'
String templateMidTableAction = rule.getTemplate().getMidTableAction().replace("\n", " ");
Map<String, String> filters = new HashMap<>(2);
if (CUSTOM_RULE.intValue() == rule.getRuleType()) {
templateMidTableAction = customMidTableActionUpdate(rule, templateMidTableAction, date, execParams, partition, ruleMetricMap);
} else if (MUL_SOURCE_RULE.intValue() == rule.getRuleType()) {
templateMidTableAction = multiMidTableActionUpdate(rule, templateMidTableAction, date, filters);
}
// Get input meta from template
List<RuleVariable> inputMetaRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.TEMPLATE_INPUT_META.getCode())).collect(Collectors.toList());
// If partition is not specified, replace with filter in rule configuration.
if (StringUtils.isBlank(partition.toString())) {
templateMidTableAction = fillPartitionWithRuleConfiguration(partition, rule, templateMidTableAction, inputMetaRuleVariables);
}
// Get dbs and tables
Map<String, String> dbTableMap = new HashMap<>(4);
// Get mappings
StringBuffer mappings = new StringBuffer();
StringBuffer realFilter = new StringBuffer();
// Get SQL From template and replace all replaceholders
String midTableAction = replaceVariable(templateMidTableAction, inputMetaRuleVariables, partition.toString(), realFilter, dbTableMap, mappings, date);
Set<TemplateStatisticsInputMeta> templateStatisticsAction = rule.getTemplate().getStatisticAction();
Map sourceConnect = new HashMap(8);
Map targetConnect = new HashMap(8);
if (dataSourceMysqlConnect != null && dataSourceMysqlConnect.size() > 0) {
for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
Map connectParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
if (connectParams == null) {
continue;
}
if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(0)) {
// If mysql sec, decrypt password and user name.
sourceConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
}
if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(1)) {
// If mysql sec, decrypt password and user name.
targetConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
}
}
}
sqlList.add("val UUID = java.util.UUID.randomUUID.toString");
// 跨表规则
if (RuleTemplateTypeEnum.MULTI_SOURCE_TEMPLATE.getCode().equals(rule.getTemplate().getTemplateType()) && dbTableMap.size() > 0) {
// Import sql function.
sqlList.addAll(getImportSql());
// Generate UUID.
// Transform original table.
Set<String> columns = new HashSet<>();
if (rule.getTemplate().getId().longValue() == MUL_SOURCE_ACCURACY_TEMPLATE_ID.longValue()) {
// Get accuracy columns.
columns = rule.getRuleDataSourceMappings().stream().map(RuleDataSourceMapping::getLeftColumnNames).map(column -> column.replace("tmp1.", "").replace("tmp2.", "")).collect(Collectors.toSet());
}
if (rule.getTemplate().getId().longValue() == MUL_SOURCE_COMMON_TEMPLATE_ID.longValue()) {
sqlList.addAll(getCommonTransformSql(dbTableMap, mappings, count, partition.toString(), filters, sourceConnect, targetConnect));
} else {
sqlList.addAll(getSpecialTransformSql(dbTableMap, count, partition.toString(), filters, Strings.join(columns, ','), sourceConnect, targetConnect));
if (optimizationConfig.getLightweightQuery()) {
count += 3;
}
}
sqlList.addAll(getSaveMidTableSentenceSettings());
sqlList.addAll(getSaveMidTableSentence(midTableName, count, runDate));
} else {
// Generate select statement and save into hive database
RuleDataSource ruleDataSource = rule.getRuleDataSources().stream().filter(dataSource -> dataSource.getDatasourceIndex() == null).iterator().next();
Map connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
if (connParams != null) {
connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
}
sqlList.addAll(generateSparkSqlAndSaveSentence(midTableAction, midTableName, rule.getTemplate(), count, connParams, runDate));
count++;
}
// Generate statistics statement, and save into mysql
List<RuleVariable> statisticsRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.STATISTICS_ARG.getCode())).collect(Collectors.toList());
sqlList.addAll(saveStatisticAndSaveMySqlSentence(rule.getId(), ruleMetricMap, templateStatisticsAction, applicationId, statisticsRuleVariables, createTime, count, runDate));
return sqlList;
}
Aggregations