use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSourceMapping in project Qualitis by WeBankFinTech.
the class RuleBatchServiceImpl method getMappingInfoAndAlarm.
private void getMappingInfoAndAlarm(Set<RuleDataSourceMapping> ruleDataSourceMappings, Set<AlarmConfig> alarmConfigs, List<ExcelMultiTemplateRuleByProject> lines, ExcelMultiTemplateRuleByProject ruleLinePrefix, String localeStr) {
for (RuleDataSourceMapping mapping : ruleDataSourceMappings) {
ExcelMultiTemplateRuleByProject tmp = new ExcelMultiTemplateRuleByProject(ruleLinePrefix.getRuleName());
tmp.setLeftMappingStatement(mapping.getLeftStatement());
tmp.setLeftMappingNames(mapping.getLeftColumnNames());
tmp.setLeftMappingTypes(mapping.getLeftColumnTypes());
tmp.setMappingOperation(MappingOperationEnum.getByCode(mapping.getOperation()).getSymbol());
tmp.setRightMappingStatement(mapping.getRightStatement());
tmp.setRightMappingNames(mapping.getRightColumnNames());
tmp.setRightMappingTypes(mapping.getRightColumnTypes());
LOGGER.info("Collect excel line: {}", tmp);
lines.add(tmp);
}
for (AlarmConfig alarmConfig : alarmConfigs) {
Double threshold = alarmConfig.getThreshold();
String alarmOutputName = alarmConfig.getTemplateOutputMeta().getOutputName();
String alarmCompareType = CompareTypeEnum.getCompareTypeName(alarmConfig.getCompareType());
String checkTemplateName = CheckTemplateEnum.getCheckTemplateName(alarmConfig.getCheckTemplate(), localeStr);
ExcelMultiTemplateRuleByProject tmp = new ExcelMultiTemplateRuleByProject(ruleLinePrefix.getRuleName());
tmp.setCompareType(alarmCompareType);
tmp.setAlarmCheckName(alarmOutputName);
tmp.setCheckTemplateName(checkTemplateName);
tmp.setThreshold(String.valueOf(threshold));
RuleMetric ruleMetric = alarmConfig.getRuleMetric();
// Recod rule metric info (unique code).
if (ruleMetric != null) {
String enCode = ruleMetric.getEnCode();
tmp.setRuleMetricEnCode(enCode);
tmp.setRuleMetricName(ruleMetric.getName());
}
tmp.setUploadRuleMetricValue(alarmConfig.getUploadRuleMetricValue());
tmp.setUploadAbnormalValue(alarmConfig.getUploadAbnormalValue());
LOGGER.info("Collect excel line: {}", tmp);
lines.add(tmp);
}
}
use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSourceMapping in project Qualitis by WeBankFinTech.
the class RuleDataSourceMappingServiceImpl method checkAndSaveRuleDataSourceMapping.
@Override
@Transactional(rollbackFor = { Exception.class })
public List<RuleDataSourceMapping> checkAndSaveRuleDataSourceMapping(List<MultiDataSourceJoinConfigRequest> mappings, Rule rule) {
List<RuleDataSourceMapping> result = new ArrayList<>();
for (MultiDataSourceJoinConfigRequest mapping : mappings) {
RuleDataSourceMapping ruleDataSourceMapping = new RuleDataSourceMapping();
ruleDataSourceMapping.setLeftStatement(mapping.getLeftStatement());
ruleDataSourceMapping.setRightStatement(mapping.getRightStatement());
ruleDataSourceMapping.setOperation(mapping.getOperation());
ruleDataSourceMapping.setLeftColumnNames(String.join(",", mapping.getLeft().stream().map(MultiDataSourceJoinColumnRequest::getColumnName).collect(Collectors.toList())));
ruleDataSourceMapping.setLeftColumnTypes(String.join("|", mapping.getLeft().stream().map(MultiDataSourceJoinColumnRequest::getColumnType).collect(Collectors.toList())));
ruleDataSourceMapping.setRightColumnNames(String.join(",", mapping.getRight().stream().map(MultiDataSourceJoinColumnRequest::getColumnName).collect(Collectors.toList())));
ruleDataSourceMapping.setRightColumnTypes(String.join("|", mapping.getRight().stream().map(MultiDataSourceJoinColumnRequest::getColumnType).collect(Collectors.toList())));
ruleDataSourceMapping.setRule(rule);
RuleDataSourceMapping savedRuleDataSourceMapping = ruleDataSourceMappingDao.saveRuleDataSourceMapping(ruleDataSourceMapping);
result.add(savedRuleDataSourceMapping);
}
return result;
}
use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSourceMapping in project Qualitis by WeBankFinTech.
the class RuleNodeServiceImpl method importChildRule.
private void importChildRule(Rule parentRule, RuleGroup ruleGroup, RuleNodeRequest ruleNodeRequest, Project projectInDb) throws IOException {
LOGGER.info("Start to import child rule.");
ObjectMapper objectMapper = new ObjectMapper();
Rule childRule = objectMapper.readValue(ruleNodeRequest.getChildRuleObject(), Rule.class);
Template childTemplate = objectMapper.readValue(ruleNodeRequest.getChildTemplateObject(), Template.class);
Set<RuleDataSource> childRuleDataSources = objectMapper.readValue(ruleNodeRequest.getChildRuleDataSourcesObject(), new TypeReference<Set<RuleDataSource>>() {
});
Set<RuleDataSourceMapping> childRuleDataSourceMappings = objectMapper.readValue(ruleNodeRequest.getChildRuleDataSourceMappingsObject(), new TypeReference<Set<RuleDataSourceMapping>>() {
});
Set<AlarmConfig> childAlarmConfigs = objectMapper.readValue(ruleNodeRequest.getChildAlarmConfigsObject(), new TypeReference<Set<AlarmConfig>>() {
});
Set<RuleVariable> childRuleVariables = objectMapper.readValue(ruleNodeRequest.getChildRuleVariablesObject(), new TypeReference<Set<RuleVariable>>() {
});
childRule.setProject(projectInDb);
childRule.setTemplate(childTemplate);
childRule.setRuleGroup(ruleGroup);
Rule savedRule = ruleDao.saveRule(childRule);
LOGGER.info("Succeed to save child rule. Rule info is {}", savedRule.toString());
List<AlarmConfig> alarmConfigList = new ArrayList<>();
for (AlarmConfig alarmConfig : childAlarmConfigs) {
alarmConfig.setRule(savedRule);
alarmConfigList.add(alarmConfig);
}
List<RuleVariable> ruleVariablesList = new ArrayList<>();
for (RuleVariable ruleVariable : childRuleVariables) {
ruleVariable.setRule(savedRule);
ruleVariablesList.add(ruleVariable);
}
List<RuleDataSource> ruleDataSourceList = new ArrayList<>();
for (RuleDataSource ruleDataSource : childRuleDataSources) {
ruleDataSource.setProjectId(projectInDb.getId());
ruleDataSource.setRule(savedRule);
ruleDataSourceList.add(ruleDataSource);
}
savedRule.setAlarmConfigs(new HashSet<>(alarmConfigDao.saveAllAlarmConfig(alarmConfigList)));
savedRule.setRuleVariables(new HashSet<>(ruleVariableDao.saveAllRuleVariable(ruleVariablesList)));
savedRule.setRuleDataSources(new HashSet<>(ruleDataSourceDao.saveAllRuleDataSource(ruleDataSourceList)));
for (RuleDataSourceMapping ruleDataSourceMapping : childRuleDataSourceMappings) {
ruleDataSourceMapping.setRule(savedRule);
ruleDataSourceMappingDao.saveRuleDataSourceMapping(ruleDataSourceMapping);
}
savedRule.setParentRule(parentRule);
parentRule.setChildRule(savedRule);
LOGGER.info("Succeed to import child rule.");
}
use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSourceMapping in project Qualitis by WeBankFinTech.
the class SqlTemplateConverter method generateSparkSqlByTask.
/**
* Convert task into scala code
* @param rule
* @param date
* @param applicationId
* @param midTableName
* @param createTime
* @param partition
* @param execParams
* @param count
* @param runDate
* @param dataSourceMysqlConnect
* @return
* @throws ConvertException
* @throws RuleVariableNotSupportException
* @throws RuleVariableNotFoundException
*/
private List<String> generateSparkSqlByTask(Rule rule, Date date, String applicationId, String midTableName, String createTime, StringBuffer partition, Map<String, String> execParams, int count, String runDate, Map<Long, Map> dataSourceMysqlConnect) throws ConvertException, RuleVariableNotSupportException, RuleVariableNotFoundException, UnExpectedRequestException {
List<String> sqlList = new ArrayList<>();
// Collect rule metric and build in save sentence sql.
List<RuleMetric> ruleMetrics = rule.getAlarmConfigs().stream().map(AlarmConfig::getRuleMetric).distinct().collect(Collectors.toList());
Map<String, Long> ruleMetricMap = new HashMap<>(ruleMetrics.size());
if (CollectionUtils.isNotEmpty(ruleMetrics)) {
LOGGER.info("Start to get rule metric for task result save. Rule metrics: {}", Arrays.toString(ruleMetrics.toArray()));
for (RuleMetric ruleMetric : ruleMetrics) {
if (ruleMetric != null) {
ruleMetricMap.put(ruleMetric.getName(), ruleMetric.getId());
}
}
LOGGER.info("Finish to get rule metric for task result save.");
}
// Get SQL from template after remove '\n'
String templateMidTableAction = rule.getTemplate().getMidTableAction().replace("\n", " ");
Map<String, String> filters = new HashMap<>(2);
if (CUSTOM_RULE.intValue() == rule.getRuleType()) {
templateMidTableAction = customMidTableActionUpdate(rule, templateMidTableAction, date, execParams, partition, ruleMetricMap);
} else if (MUL_SOURCE_RULE.intValue() == rule.getRuleType()) {
templateMidTableAction = multiMidTableActionUpdate(rule, templateMidTableAction, date, filters);
}
// Get input meta from template
List<RuleVariable> inputMetaRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.TEMPLATE_INPUT_META.getCode())).collect(Collectors.toList());
// If partition is not specified, replace with filter in rule configuration.
if (StringUtils.isBlank(partition.toString())) {
templateMidTableAction = fillPartitionWithRuleConfiguration(partition, rule, templateMidTableAction, inputMetaRuleVariables);
}
// Get dbs and tables
Map<String, String> dbTableMap = new HashMap<>(4);
// Get mappings
StringBuffer mappings = new StringBuffer();
StringBuffer realFilter = new StringBuffer();
// Get SQL From template and replace all replaceholders
String midTableAction = replaceVariable(templateMidTableAction, inputMetaRuleVariables, partition.toString(), realFilter, dbTableMap, mappings, date);
Set<TemplateStatisticsInputMeta> templateStatisticsAction = rule.getTemplate().getStatisticAction();
Map sourceConnect = new HashMap(8);
Map targetConnect = new HashMap(8);
if (dataSourceMysqlConnect != null && dataSourceMysqlConnect.size() > 0) {
for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
Map connectParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
if (connectParams == null) {
continue;
}
if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(0)) {
// If mysql sec, decrypt password and user name.
sourceConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
}
if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(1)) {
// If mysql sec, decrypt password and user name.
targetConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
}
}
}
sqlList.add("val UUID = java.util.UUID.randomUUID.toString");
// 跨表规则
if (RuleTemplateTypeEnum.MULTI_SOURCE_TEMPLATE.getCode().equals(rule.getTemplate().getTemplateType()) && dbTableMap.size() > 0) {
// Import sql function.
sqlList.addAll(getImportSql());
// Generate UUID.
// Transform original table.
Set<String> columns = new HashSet<>();
if (rule.getTemplate().getId().longValue() == MUL_SOURCE_ACCURACY_TEMPLATE_ID.longValue()) {
// Get accuracy columns.
columns = rule.getRuleDataSourceMappings().stream().map(RuleDataSourceMapping::getLeftColumnNames).map(column -> column.replace("tmp1.", "").replace("tmp2.", "")).collect(Collectors.toSet());
}
if (rule.getTemplate().getId().longValue() == MUL_SOURCE_COMMON_TEMPLATE_ID.longValue()) {
sqlList.addAll(getCommonTransformSql(dbTableMap, mappings, count, partition.toString(), filters, sourceConnect, targetConnect));
} else {
sqlList.addAll(getSpecialTransformSql(dbTableMap, count, partition.toString(), filters, Strings.join(columns, ','), sourceConnect, targetConnect));
if (optimizationConfig.getLightweightQuery()) {
count += 3;
}
}
sqlList.addAll(getSaveMidTableSentenceSettings());
sqlList.addAll(getSaveMidTableSentence(midTableName, count, runDate));
} else {
// Generate select statement and save into hive database
RuleDataSource ruleDataSource = rule.getRuleDataSources().stream().filter(dataSource -> dataSource.getDatasourceIndex() == null).iterator().next();
Map connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
if (connParams != null) {
connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
}
sqlList.addAll(generateSparkSqlAndSaveSentence(midTableAction, midTableName, rule.getTemplate(), count, connParams, runDate));
count++;
}
// Generate statistics statement, and save into mysql
List<RuleVariable> statisticsRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.STATISTICS_ARG.getCode())).collect(Collectors.toList());
sqlList.addAll(saveStatisticAndSaveMySqlSentence(rule.getId(), ruleMetricMap, templateStatisticsAction, applicationId, statisticsRuleVariables, createTime, count, runDate));
return sqlList;
}
Aggregations