use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceConfigRequest in project Qualitis by WeBankFinTech.
the class RuleNodeServiceImpl method constructMultiRequest.
private AddMultiSourceRuleRequest constructMultiRequest(Rule rule, RuleGroup ruleGroup) {
AddMultiSourceRuleRequest addMultiSourceRuleRequest = new AddMultiSourceRuleRequest();
String newVersion = ruleGroup.getVersion();
if (StringUtils.isEmpty(newVersion)) {
String newRuleName = rule.getName() + "_copy_" + ruleGroup.getId();
LOGGER.info("Multi rule start to be copied. Copied rule name: " + newRuleName);
addMultiSourceRuleRequest.setRuleName(newRuleName);
if (StringUtils.isNotBlank(rule.getCnName())) {
addMultiSourceRuleRequest.setRuleCnName(rule.getCnName() + "_副本");
}
} else {
String oldVersion = rule.getRuleGroup().getVersion();
LOGGER.info("Multi rule start to be copied. Copied rule name: " + rule.getName() + "_" + ruleGroup.getVersion());
if (StringUtils.isNotBlank(oldVersion) && rule.getName().endsWith(oldVersion)) {
addMultiSourceRuleRequest.setRuleName(rule.getName().replace(oldVersion, newVersion));
if (StringUtils.isNotBlank(rule.getCnName())) {
addMultiSourceRuleRequest.setRuleCnName(rule.getCnName().replace(oldVersion, newVersion));
}
} else {
addMultiSourceRuleRequest.setRuleName(rule.getName() + "_" + newVersion);
if (StringUtils.isNotBlank(rule.getCnName())) {
addMultiSourceRuleRequest.setRuleCnName(rule.getCnName() + "_" + newVersion);
}
}
}
String ruleDetail = rule.getDetail();
String clusterName = rule.getRuleDataSources().iterator().next().getClusterName();
addMultiSourceRuleRequest.setAbortOnFailure(rule.getAbortOnFailure());
addMultiSourceRuleRequest.setClusterName(clusterName);
addMultiSourceRuleRequest.setRuleDetail(ruleDetail);
addMultiSourceRuleRequest.setCsId(rule.getCsId());
addMultiSourceRuleRequest.setSpecifyStaticStartupParam(rule.getSpecifyStaticStartupParam());
addMultiSourceRuleRequest.setDeleteFailCheckResult(rule.getDeleteFailCheckResult());
addMultiSourceRuleRequest.setMultiSourceRuleTemplateId(rule.getTemplate().getId());
addMultiSourceRuleRequest.setStaticStartupParam(rule.getStaticStartupParam());
addMultiSourceRuleRequest.setProjectId(ruleGroup.getProjectId());
addMultiSourceRuleRequest.setRuleGroupId(ruleGroup.getId());
List<RuleVariable> filterRuleVariable = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getTemplateMidTableInputMeta().getInputType().equals(TemplateInputTypeEnum.CONDITION.getCode())).collect(Collectors.toList());
if (filterRuleVariable != null && filterRuleVariable.size() != 0) {
addMultiSourceRuleRequest.setFilter(filterRuleVariable.iterator().next().getValue());
}
// Data source request
for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
String databaseName = ruleDataSource.getDbName();
String tableName = ruleDataSource.getTableName();
String filter = ruleDataSource.getFilter();
Integer datasourceIndex = ruleDataSource.getDatasourceIndex();
MultiDataSourceConfigRequest dataSourceConfigRequest = new MultiDataSourceConfigRequest();
dataSourceConfigRequest.setDbName(databaseName);
dataSourceConfigRequest.setTableName(tableName);
dataSourceConfigRequest.setFilter(filter);
dataSourceConfigRequest.setProxyUser(ruleDataSource.getProxyUser());
if (ruleDataSource.getLinkisDataSourceId() != null) {
dataSourceConfigRequest.setLinkisDataSourceType(TemplateDataSourceTypeEnum.getMessage(ruleDataSource.getDatasourceType()));
dataSourceConfigRequest.setLinkisDataSourceName(ruleDataSource.getLinkisDataSourceName());
dataSourceConfigRequest.setLinkisDataSourceId(ruleDataSource.getLinkisDataSourceId());
}
if (datasourceIndex == 0) {
addMultiSourceRuleRequest.setSource(dataSourceConfigRequest);
} else {
addMultiSourceRuleRequest.setTarget(dataSourceConfigRequest);
}
}
// Mapping
List<MultiDataSourceJoinConfigRequest> mappings = new ArrayList<>();
for (RuleDataSourceMapping mapping : rule.getRuleDataSourceMappings()) {
MultiDataSourceJoinConfigRequest multiDataSourceJoinConfigRequest = new MultiDataSourceJoinConfigRequest();
multiDataSourceJoinConfigRequest.setOperation(mapping.getOperation());
multiDataSourceJoinConfigRequest.setLeftStatement(mapping.getLeftStatement());
multiDataSourceJoinConfigRequest.setRightStatement(mapping.getRightStatement());
List<MultiDataSourceJoinColumnRequest> left = getMultiDataSourceJoinColumnRequest(mapping.getLeftColumnNames(), mapping.getLeftColumnTypes());
List<MultiDataSourceJoinColumnRequest> right = getMultiDataSourceJoinColumnRequest(mapping.getRightColumnNames(), mapping.getLeftColumnTypes());
multiDataSourceJoinConfigRequest.setLeft(left);
multiDataSourceJoinConfigRequest.setRight(right);
mappings.add(multiDataSourceJoinConfigRequest);
}
addMultiSourceRuleRequest.setMappings(mappings);
List<AlarmConfigRequest> alarmConfigRequests = constructAlarmConfigRequest(rule.getAlarmConfigs());
addMultiSourceRuleRequest.setAlarm(true);
addMultiSourceRuleRequest.setAlarmVariable(alarmConfigRequests);
return addMultiSourceRuleRequest;
}
use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceConfigRequest in project Qualitis by WeBankFinTech.
the class MetaDataServiceImpl method constructRequest.
private AddMultiSourceRuleRequest constructRequest(MulDbRequest request, FilterRequest filterRequest, String currentSourceTable, String currentTargetTable, int index, int total, String loginUser) throws UnExpectedRequestException, MetaDataAcquireFailedException, IOException {
LOGGER.info("Start to construct add multi source rule request.");
// rule basic info.
AddMultiSourceRuleRequest addMultiSourceRuleRequest = new AddMultiSourceRuleRequest();
basicInfo(addMultiSourceRuleRequest, request, index, total);
MultiDataSourceConfigRequest sourceConfigRequest = new MultiDataSourceConfigRequest();
sourceConfigRequest.setLinkisDataSourceName(request.getSourceLinkisDataSourceName());
sourceConfigRequest.setLinkisDataSourceType(request.getSourceLinkisDataSourceType());
sourceConfigRequest.setLinkisDataSourceId(request.getSourceLinkisDataSourceId());
sourceConfigRequest.setProxyUser(request.getProxyUser());
sourceConfigRequest.setDbName(request.getSourceDb());
sourceConfigRequest.setContextService(false);
MultiDataSourceConfigRequest targetConfigRequest = new MultiDataSourceConfigRequest();
targetConfigRequest.setLinkisDataSourceName(request.getTargetLinkisDataSourceName());
targetConfigRequest.setLinkisDataSourceType(request.getTargetLinkisDataSourceType());
targetConfigRequest.setLinkisDataSourceId(request.getTargetLinkisDataSourceId());
targetConfigRequest.setProxyUser(request.getProxyUser());
targetConfigRequest.setDbName(request.getTargetDb());
targetConfigRequest.setContextService(false);
if (filterRequest == null) {
// source table and target table
sourceConfigRequest.setTableName(currentSourceTable);
sourceConfigRequest.setFilter("true");
targetConfigRequest.setTableName(currentTargetTable);
targetConfigRequest.setFilter("true");
addMultiSourceRuleRequest.setMultiSourceRuleTemplateId(20L);
List<AlarmConfigRequest> alarmConfigRequests = new ArrayList<>();
alarmConfigRequests.add(new AlarmConfigRequest(645L, CheckTemplateEnum.FIXED_VALUE.getCode(), CompareTypeEnum.EQUAL.getCode(), 0.0));
addMultiSourceRuleRequest.setAlarmVariable(alarmConfigRequests);
} else {
sourceConfigRequest.setTableName(filterRequest.getSourceTable());
if (StringUtils.isNotBlank(filterRequest.getSourceFilter())) {
sourceConfigRequest.setFilter("! (" + filterRequest.getSourceFilter() + ")");
} else {
sourceConfigRequest.setFilter("true");
}
targetConfigRequest.setTableName(filterRequest.getTargetTable());
if (StringUtils.isNotBlank(filterRequest.getTargetFilter())) {
targetConfigRequest.setFilter("! (" + filterRequest.getTargetFilter() + ")");
} else {
targetConfigRequest.setFilter("true");
}
addMultiSourceRuleRequest.setMultiSourceRuleTemplateId(17L);
List<AlarmConfigRequest> alarmConfigRequests = new ArrayList<>();
alarmConfigRequests.add(new AlarmConfigRequest(33L, CheckTemplateEnum.FIXED_VALUE.getCode(), CompareTypeEnum.EQUAL.getCode(), 0.0));
addMultiSourceRuleRequest.setAlarmVariable(alarmConfigRequests);
}
addMultiSourceRuleRequest.setSource(sourceConfigRequest);
addMultiSourceRuleRequest.setTarget(targetConfigRequest);
// All fields mappings
LOGGER.info("Start to get all fields with source database's table and target database's table.");
List<ColumnInfoDetail> sourceFields;
List<ColumnInfoDetail> targetFields;
if (request.getSourceLinkisDataSourceId() == null) {
GetUserColumnByTableIdRequest getUserColumnSourceRequest = new GetUserColumnByTableIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getSourceDb(), filterRequest == null ? currentSourceTable : filterRequest.getSourceTable());
getUserColumnSourceRequest.setProxyUser(request.getProxyUser());
getUserColumnSourceRequest.setLoginUser(loginUser);
sourceFields = getUserColumnByTableId(getUserColumnSourceRequest).getData().getData();
} else {
sourceFields = getColumnsByDataSource(request.getClusterName(), request.getProxyUser(), request.getSourceLinkisDataSourceId(), request.getSourceDb(), currentSourceTable).getData().getData();
}
if (request.getTargetLinkisDataSourceId() == null) {
GetUserColumnByTableIdRequest getUserColumnTargetRequest = new GetUserColumnByTableIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getTargetDb(), filterRequest == null ? currentTargetTable : filterRequest.getTargetTable());
getUserColumnTargetRequest.setProxyUser(request.getProxyUser());
getUserColumnTargetRequest.setLoginUser(loginUser);
targetFields = getUserColumnByTableId(getUserColumnTargetRequest).getData().getData();
} else {
targetFields = getColumnsByDataSource(request.getClusterName(), request.getProxyUser(), request.getTargetLinkisDataSourceId(), request.getTargetDb(), currentTargetTable).getData().getData();
}
if (CollectionUtils.isEmpty(sourceFields) || CollectionUtils.isEmpty(targetFields)) {
throw new UnExpectedRequestException("There is table which has none field");
}
LOGGER.info("Success to get all fields with source database's table and target database's table.");
// Sorted columns.
Collections.sort(sourceFields, new Comparator<ColumnInfoDetail>() {
@Override
public int compare(ColumnInfoDetail front, ColumnInfoDetail back) {
return front.getFieldName().compareTo(back.getFieldName());
}
});
Collections.sort(targetFields, new Comparator<ColumnInfoDetail>() {
@Override
public int compare(ColumnInfoDetail front, ColumnInfoDetail back) {
return front.getFieldName().compareTo(back.getFieldName());
}
});
String sourceFieldStr = sourceFields.stream().map(ColumnInfoDetail::getFieldName).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
String sourceFieldTypeStr = sourceFields.stream().map(ColumnInfoDetail::getDataType).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
String targetFieldStr = targetFields.stream().map(ColumnInfoDetail::getFieldName).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
String targetFieldTypeStr = targetFields.stream().map(ColumnInfoDetail::getDataType).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
if (sourceFields.size() == 0 || targetFields.size() == 0 || sourceFields.size() != targetFields.size() || !sourceFieldStr.equals(targetFieldStr) || !sourceFieldTypeStr.equals(targetFieldTypeStr)) {
throw new UnExpectedRequestException("Create multi-db rules failed, because the fields' name of table is different.");
}
List<MultiDataSourceJoinConfigRequest> mappings = new ArrayList<>(sourceFields.size());
for (int j = 0; j < sourceFields.size() && j < targetFields.size(); j++) {
ColumnInfoDetail currentSourceField = sourceFields.get(j);
ColumnInfoDetail currentTargetField = targetFields.get(j);
if (filterRequest != null && CollectionUtils.isNotEmpty(filterRequest.getFilterColumnList()) && filterRequest.getFilterColumnList().contains(currentSourceField.getFieldName())) {
continue;
}
if (currentSourceField.getFieldName().equals(currentTargetField.getFieldName()) && currentSourceField.getDataType().equals(currentTargetField.getDataType())) {
// TODO: continue
MultiDataSourceJoinConfigRequest multiDataSourceJoinConfigRequest = new MultiDataSourceJoinConfigRequest();
MultiDataSourceJoinColumnRequest leftJoinColumnRequest = new MultiDataSourceJoinColumnRequest("tmp1.".concat(currentSourceField.getFieldName()), currentSourceField.getDataType());
MultiDataSourceJoinColumnRequest rightJoinColumnRequest = new MultiDataSourceJoinColumnRequest("tmp2.".concat(currentTargetField.getFieldName()), currentTargetField.getDataType());
multiDataSourceJoinConfigRequest.setOperation(MappingOperationEnum.EQUAL.getCode());
multiDataSourceJoinConfigRequest.setLeft(Arrays.asList(leftJoinColumnRequest));
multiDataSourceJoinConfigRequest.setRight(Arrays.asList(rightJoinColumnRequest));
multiDataSourceJoinConfigRequest.setLeftStatement("tmp1.".concat(currentSourceField.getFieldName()));
multiDataSourceJoinConfigRequest.setRightStatement("tmp2.".concat(currentTargetField.getFieldName()));
mappings.add(multiDataSourceJoinConfigRequest);
} else {
throw new UnExpectedRequestException("Create multi-db rules failed, because the fields' name of tables is different");
}
}
addMultiSourceRuleRequest.setMappings(mappings);
addMultiSourceRuleRequest.setLoginUser(loginUser);
LOGGER.info("Success to construct add multi source rule request. Request[{}]", new ObjectMapper().writeValueAsString(addMultiSourceRuleRequest));
return addMultiSourceRuleRequest;
}
use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceConfigRequest in project Qualitis by WeBankFinTech.
the class AddMultiRuleRequestBuilder method solveDatasource.
private Map<String, List<ColumnInfoDetail>> solveDatasource(StringBuffer cluster, String datasource) throws UnExpectedRequestException, MetaDataAcquireFailedException {
MultiDataSourceConfigRequest source;
MultiDataSourceConfigRequest target;
List<MultiDataSourceJoinConfigRequest> mappings = new ArrayList<>();
String[] datasourceStrs = datasource.split(SpecCharEnum.VERTICAL_BAR.getValue());
String sourcStr = datasourceStrs[0];
String targetStr = datasourceStrs[1];
Matcher matcherName = DATA_SOURCE_NAME.matcher(cluster.toString().toUpperCase());
Matcher matcherId = DATA_SOURCE_ID.matcher(cluster.toString().toUpperCase());
String dataSourceName = "";
String dataSourceId = "";
while (matcherId.find()) {
String group = matcherId.group();
dataSourceId = group.replace(".(", "").replace(")", "").split("=")[1];
int startIndex = cluster.toString().toUpperCase().indexOf(group);
String replaceStr = cluster.substring(startIndex, startIndex + group.length());
cluster.delete(startIndex, startIndex + group.length());
}
if (StringUtils.isBlank(dataSourceId)) {
while (matcherName.find()) {
String group = matcherName.group();
int startIndex = cluster.toString().toUpperCase().indexOf(group);
String replaceStr = cluster.substring(startIndex, startIndex + group.length());
dataSourceName = replaceStr.replace(".(", "").replace(")", "").split("=")[1];
cluster.delete(startIndex, startIndex + group.length());
}
}
source = toRequest(cluster.toString(), sourcStr, dataSourceId, dataSourceName);
target = toRequest(cluster.toString(), targetStr, dataSourceId, dataSourceName);
addMultiSourceRuleRequest.setSource(source);
addMultiSourceRuleRequest.setTarget(target);
addMultiSourceRuleRequest.setMappings(mappings);
if (datasourceStrs.length > TWO) {
throw new UnExpectedRequestException("Datasource param is illegle");
}
List<ColumnInfoDetail> leftCols = metaDataClient.getColumnInfo(cluster.toString(), source.getDbName(), source.getTableName(), userName);
List<ColumnInfoDetail> rightCols = metaDataClient.getColumnInfo(cluster.toString(), target.getDbName(), target.getTableName(), userName);
Map<String, List<ColumnInfoDetail>> colsMap = new HashMap<>(2);
colsMap.put(TMP_1, leftCols);
colsMap.put(TMP_2, rightCols);
return colsMap;
}
use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceConfigRequest in project Qualitis by WeBankFinTech.
the class AddMultiRuleRequestBuilder method toRequest.
private MultiDataSourceConfigRequest toRequest(String clusterName, String dataSourcStr, String dataSourceId, String dataSourceName) throws UnExpectedRequestException, MetaDataAcquireFailedException {
String[] datasourceStrs = dataSourcStr.split(SpecCharEnum.COLON.getValue());
if (datasourceStrs.length > TWO) {
throw new UnExpectedRequestException("Datasource param is illegle");
}
String[] datasources = datasourceStrs[0].split(SpecCharEnum.PERIOD.getValue());
String filter = datasourceStrs[1];
if (datasources.length != TWO) {
throw new UnExpectedRequestException("Datasource param is illegle");
}
String database = datasources[0];
String table = datasources[1];
if (StringUtils.isNotBlank(dataSourceId)) {
LOGGER.info("Find data source connect. Data source ID: " + dataSourceId);
Long realDataSourceId = Long.parseLong(dataSourceId);
GeneralResponse<Map> response = metaDataClient.getDataSourceInfoDetail(clusterName, userName, realDataSourceId, null);
Map dataSourceInfo = ((Map) response.getData().get("info"));
String dataSourceInfoName = (String) dataSourceInfo.get("dataSourceName");
String dataSourceInfoType = (String) ((Map) dataSourceInfo.get("dataSourceType")).get("name");
return new MultiDataSourceConfigRequest(database, table, filter, realDataSourceId, dataSourceInfoName, dataSourceInfoType);
} else if (StringUtils.isNotBlank(dataSourceName)) {
LOGGER.info("Find data source connect. Data source name: " + dataSourceName);
GeneralResponse<Map> response = metaDataClient.getDataSourceInfoDetailByName(clusterName, userName, dataSourceName);
Map dataSourceInfo = ((Map) response.getData().get("info"));
String dataSourceInfoName = (String) dataSourceInfo.get("dataSourceName");
String dataSourceInfoType = (String) ((Map) dataSourceInfo.get("dataSourceType")).get("name");
Integer currentDataSourceId = (Integer) dataSourceInfo.get("id");
return new MultiDataSourceConfigRequest(database, table, filter, currentDataSourceId.longValue(), dataSourceInfoName, dataSourceInfoType);
}
return new MultiDataSourceConfigRequest(database, table, filter);
}
use of com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceConfigRequest in project Qualitis by WeBankFinTech.
the class RuleBatchServiceImpl method constructAddMultiSourceRuleRequest.
private List<AddMultiSourceRuleRequest> constructAddMultiSourceRuleRequest(Map<String, List<ExcelMultiTemplateRuleByProject>> multiRulePartitionedByRuleName, Project project, String localeStr) throws UnExpectedRequestException {
List<AddMultiSourceRuleRequest> addMultiSourceRuleRequests = new ArrayList<>();
for (String ruleName : multiRulePartitionedByRuleName.keySet()) {
List<ExcelMultiTemplateRuleByProject> ruleInfos = multiRulePartitionedByRuleName.get(ruleName);
ExcelMultiTemplateRuleByProject firstCommonInfo = ruleInfos.get(0);
AddMultiSourceRuleRequest addMultiSourceRuleRequest = new AddMultiSourceRuleRequest();
String ruleTemplateName = firstCommonInfo.getTemplateName();
String ruleGroupName = firstCommonInfo.getRuleGroupName();
addMultiSourceRuleRequest.setSpecifyStaticStartupParam(firstCommonInfo.getSpecifyStaticStartupParam());
addMultiSourceRuleRequest.setDeleteFailCheckResult(firstCommonInfo.getDeleteFailCheckResult());
addMultiSourceRuleRequest.setStaticStartupParam(firstCommonInfo.getStaticStartupParam());
addMultiSourceRuleRequest.setAbortOnFailure(firstCommonInfo.getAbortOnFailure());
if (StringUtils.isBlank(ruleGroupName)) {
throw new UnExpectedRequestException("RuleGroupName {&CAN_NOT_BE_NULL_OR_EMPTY}");
}
Template template = findTemplateByName(ruleTemplateName);
if (template == null) {
throw new UnExpectedRequestException("{&TEMPLATE_NAME}: [" + ruleTemplateName + "] {&DOES_NOT_EXIST}");
}
String filter = null;
boolean alarm = false;
String clusterName = firstCommonInfo.getClusterName();
MultiDataSourceConfigRequest sourceConfigRequest = new MultiDataSourceConfigRequest();
MultiDataSourceConfigRequest targetConfigRequest = new MultiDataSourceConfigRequest();
List<MultiDataSourceJoinConfigRequest> mappings = new ArrayList<>();
List<AlarmConfigRequest> alarmConfigRequests = new ArrayList<>();
for (ExcelMultiTemplateRuleByProject excelMultiTemplateRule : ruleInfos) {
if (StringUtils.isNotBlank(excelMultiTemplateRule.getWhereFilter())) {
filter = excelMultiTemplateRule.getWhereFilter();
}
getAlarmConfig(alarmConfigRequests, excelMultiTemplateRule, template, localeStr);
getMultiDataSourceRequest(sourceConfigRequest, excelMultiTemplateRule, 0);
getMultiDataSourceRequest(targetConfigRequest, excelMultiTemplateRule, 1);
getMultiDataSourceJoinRequest(mappings, excelMultiTemplateRule);
}
if (alarmConfigRequests.size() != 0) {
alarm = true;
}
RuleGroup ruleGroupInDb = ruleGroupDao.findByRuleGroupNameAndProjectId(ruleGroupName, project.getId());
if (ruleGroupInDb != null) {
addMultiSourceRuleRequest.setRuleGroupId(ruleGroupInDb.getId());
} else {
RuleGroup ruleGroup = ruleGroupDao.saveRuleGroup(new RuleGroup(ruleGroupName, project.getId()));
addMultiSourceRuleRequest.setRuleGroupId(ruleGroup.getId());
}
addMultiSourceRuleRequest.setRuleName(ruleName);
addMultiSourceRuleRequest.setClusterName(clusterName);
addMultiSourceRuleRequest.setProjectId(project.getId());
addMultiSourceRuleRequest.setRuleDetail(firstCommonInfo.getRuleDetail());
addMultiSourceRuleRequest.setMultiSourceRuleTemplateId(template.getId());
addMultiSourceRuleRequest.setAlarmVariable(alarmConfigRequests);
addMultiSourceRuleRequests.add(addMultiSourceRuleRequest);
addMultiSourceRuleRequest.setSource(sourceConfigRequest);
addMultiSourceRuleRequest.setTarget(targetConfigRequest);
addMultiSourceRuleRequest.setMappings(mappings);
addMultiSourceRuleRequest.setFilter(filter);
addMultiSourceRuleRequest.setAlarm(alarm);
}
return addMultiSourceRuleRequests;
}
Aggregations