use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.
the class RuleQueryServiceImpl method getProjectsByUserPerm.
private List<RuleQueryProject> getProjectsByUserPerm(DataSourceQo param, List<ProjectUser> projectUsers, Map<Long, RuleQueryProject> projectMap) {
if (projectUsers == null || projectUsers.isEmpty()) {
return null;
}
for (ProjectUser projectUser : projectUsers) {
List<RuleDataSource> projectDataSources = ruleDataSourceDao.findByProjectUser(projectUser.getProject().getId(), param.getCluster(), param.getDb(), param.getTable());
if (projectDataSources == null || projectDataSources.isEmpty()) {
continue;
}
addRuleDataSource(projectDataSources, projectMap, projectUser.getProject());
}
return new ArrayList<>(projectMap.values());
}
use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method commonExecution.
@Override
public ApplicationTaskSimpleResponse commonExecution(List<Rule> rules, StringBuffer partition, String executionUser, String nodeName, String startupParam, String clusterName, String setFlag, Map<String, String> execParams, Application newApplication, Date date, StringBuffer runDate) throws RuleVariableNotSupportException, JobSubmitException, RuleVariableNotFoundException, ArgumentException, ConvertException, DataQualityTaskException, TaskTypeException, ClusterInfoNotConfigException, SystemConfigException, UnExpectedRequestException, MetaDataAcquireFailedException, IOException, TaskNotExistException, SemanticException, ParseException, DataSourceMoveException, DataSourceOverSizeException, org.apache.hadoop.hive.ql.parse.ParseException, BothNullDatasourceException, RightNullDatasourceException, LeftNullDatasourceException, java.text.ParseException {
// current user
String userName = executionUser;
// Generate database name.
String database = generateDatabase(userName);
LOGGER.info("Succeed to generate database_name: {}", database);
// Save application
newApplication.setRuleSize(rules.size());
newApplication.setSavedDb(database);
Application saveApplication = applicationDao.saveApplication(newApplication);
List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
List<Rule> fileRules = new ArrayList<>();
Map<Long, Map> dataSourceMysqlConnect = new HashMap<>(2);
for (Iterator<Rule> iterator = rules.iterator(); iterator.hasNext(); ) {
Rule currentRule = iterator.next();
if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
// Replace with execution parameter and parse datasource to save.
customReSaveDateSource(currentRule, execParams, clusterName, date);
}
List<Map<String, String>> mappingCols = new ArrayList<>();
getMappingCols(currentRule, mappingCols);
// Check datasource before submit job.
try {
checkDatasource(currentRule, userName, partition, mappingCols, nodeName, clusterName, dataSourceMysqlConnect);
} catch (BothNullDatasourceException e) {
Task taskInDb = taskDao.save(new Task(newApplication, newApplication.getSubmitTime(), TaskStatusEnum.SUCCEED.getCode()));
taskInDb.setClusterName(clusterName);
TaskRuleSimple taskRuleSimple = new TaskRuleSimple(currentRule, taskInDb);
Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
}
taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
taskInDb.setTaskDataSources(taskDataSources);
taskInDb.setTaskRuleSimples(taskRuleSimples);
TaskResult taskResult = new TaskResult();
taskResult.setApplicationId(newApplication.getId());
taskResult.setCreateTime(newApplication.getSubmitTime());
taskResult.setRuleId(currentRule.getId());
taskResult.setResultType("Long");
taskResult.setValue(0 + "");
taskResultDao.saveTaskResult(taskResult);
taskDao.save(taskInDb);
iterator.remove();
if (!iterator.hasNext()) {
throw e;
}
continue;
}
if (currentRule.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())) {
fileRules.add(currentRule);
LOGGER.info("Succeed to find file rule. Rule: {}", currentRule.getId() + " " + currentRule.getName());
iterator.remove();
}
}
String submitTime = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(date);
// General task.
if (!rules.isEmpty()) {
taskSubmitResults.addAll(executionManager.submitApplication(rules, nodeName, submitTime, userName, database, partition, date, saveApplication, clusterName, startupParam, setFlag, execParams, runDate, dataSourceMysqlConnect));
}
// Execute file rule task and save task result.
if (!fileRules.isEmpty()) {
taskSubmitResults.add(executionManager.executeFileRule(fileRules, submitTime, saveApplication, userName, clusterName, runDate));
}
saveApplication.setTotalTaskNum(taskSubmitResults.size());
LOGGER.info("Succeed to submit application. result: {}", taskSubmitResults);
Application applicationInDb = applicationDao.saveApplication(saveApplication);
LOGGER.info("Succeed to save application. application: {}", applicationInDb);
return new ApplicationTaskSimpleResponse(taskSubmitResults);
}
use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method checkDatasource.
private void checkDatasource(Rule currentRule, String userName, StringBuffer partition, List<Map<String, String>> mappingCols, String nodeName, String clusterName, Map<Long, Map> dataSourceMysqlConnect) throws UnExpectedRequestException, MetaDataAcquireFailedException, DataSourceOverSizeException, DataSourceMoveException, BothNullDatasourceException, LeftNullDatasourceException, RightNullDatasourceException {
// For multi source rule to check tables' size before submit.
List<Double> datasourceSizeList = new ArrayList<>(currentRule.getRuleDataSources().size());
for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
Map<String, String> mappingCol = null;
if (ORIGINAL_INDEX.equals(ruleDataSource.getDatasourceIndex())) {
continue;
}
if (ruleDataSource.getDatasourceIndex() != null && mappingCols.get(ruleDataSource.getDatasourceIndex()).size() > 0) {
mappingCol = mappingCols.get(ruleDataSource.getDatasourceIndex());
}
if (ruleDataSource.getLinkisDataSourceId() != null) {
LOGGER.info("Start to solve relationship datasource info.");
checkRdmsSqlMetaInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource, mappingCol);
GeneralResponse<Map> dataSourceInfoDetail = metaDataClient.getDataSourceInfoDetail(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource.getLinkisDataSourceId(), ruleDataSource.getLinkisDataSourceVersionId());
GeneralResponse<Map> dataSourceConnectParams = metaDataClient.getDataSourceConnectParams(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource.getLinkisDataSourceId(), ruleDataSource.getLinkisDataSourceVersionId());
Map connectParamsReal = (Map) dataSourceConnectParams.getData().get("connectParams");
if (connectParamsReal.size() == 0) {
throw new UnExpectedRequestException("{&THE_DATASOURCE_IS_NOT_DEPLOYED}");
}
Map connectParams = (Map) ((Map) dataSourceInfoDetail.getData().get("info")).get("connectParams");
String dataType = (String) ((Map) ((Map) dataSourceInfoDetail.getData().get("info")).get("dataSourceType")).get("name");
connectParams.put("dataType", dataType);
dataSourceMysqlConnect.put(ruleDataSource.getId(), connectParams);
continue;
}
// Parse filter fields.
List<String> filterFields = getFilterFields(partition.toString());
if (StringUtils.isNotBlank(ruleDataSource.getDbName()) && !ruleDataSource.getDbName().equals(RuleConstraintEnum.CUSTOM_DATABASE_PREFIS.getValue())) {
// Get actual fields info.
List<ColumnInfoDetail> cols = metaDataClient.getColumnInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), userName);
if (CollectionUtils.isEmpty(cols)) {
throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "]. {&RULE_DATASOURCE_BE_MOVED}");
}
// Get actual partition fields.
List<String> partitionFields = cols.stream().filter(ColumnInfoDetail::getPartitionField).map(ColumnInfoDetail::getFieldName).collect(Collectors.toList());
// Check filter fields.
boolean partitionTable = CollectionUtils.isNotEmpty(partitionFields);
if (partitionTable && partition.length() > 0) {
for (String filter : filterFields) {
if (!partitionFields.contains(filter)) {
throw new UnExpectedRequestException("Table[" + ruleDataSource.getTableName() + "]. {&THE_CHECK_FIELD_DOES_NOT_EXIST_IN_PARTITIONS}[" + filter + "]");
}
}
// Check partition size.
PartitionStatisticsInfo partitionStatisticsInfo = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(partition.toString()), userName);
String fullSize = partitionStatisticsInfo.getPartitionSize();
ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
if (clusterInfo != null && StringUtils.isNotBlank(clusterInfo.getSkipDataSize()) && StringUtils.isNotBlank(fullSize)) {
double number = 0;
String unit = "B";
if (!"0B".equals(fullSize)) {
number = Double.parseDouble(fullSize.split(" ")[0]);
unit = fullSize.split(" ")[1];
}
datasourceSizeList.add(number);
String[] skipDataSize = clusterInfo.getSkipDataSize().split(" ");
double res = UnitTransfer.alarmconfigToTaskResult(number, skipDataSize[1], unit);
LOGGER.info("Check datasource[" + fullSize + "] if or not oversize with system config[" + clusterInfo.getSkipDataSize() + "]");
if (res > Double.parseDouble(skipDataSize[0])) {
throw new DataSourceOverSizeException("Table[" + ruleDataSource.getTableName() + "]. {&TABLE_IS_OVERSIZE_WITH_SYSTEM_CONFIG}:[" + clusterInfo.getSkipDataSize() + "]");
}
}
} else {
// Check table size.
TableStatisticsInfo tableStatisticsInfo = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), userName);
String fullSize = tableStatisticsInfo.getTableSize();
if (NULL_TABLE_SIZE.equals(fullSize)) {
throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "] {&RULE_DATASOURCE_BE_MOVED}");
}
ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
if (clusterInfo != null && StringUtils.isNotBlank(clusterInfo.getSkipDataSize()) && StringUtils.isNotBlank(fullSize)) {
LOGGER.info("Check datasource[" + fullSize + "] if or not oversize with system config[" + clusterInfo.getSkipDataSize() + "]");
double number = 0;
String unit = "B";
if (!"0B".equals(fullSize)) {
number = Double.parseDouble(fullSize.split(" ")[0]);
unit = fullSize.split(" ")[1];
}
datasourceSizeList.add(number);
String[] skipDataSize = clusterInfo.getSkipDataSize().split(" ");
double res = UnitTransfer.alarmconfigToTaskResult(number, skipDataSize[1], unit);
if (res > Double.parseDouble(skipDataSize[0])) {
throw new DataSourceOverSizeException("Table[" + ruleDataSource.getTableName() + "] is oversize with system config:[" + clusterInfo.getSkipDataSize() + "]");
}
}
partition.delete(0, partition.length());
}
if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
continue;
}
if (!metaDataClient.fieldExist(ruleDataSource.getColName(), cols, mappingCol)) {
throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "] {&RULE_DATASOURCE_BE_MOVED}");
}
} else if (StringUtils.isNotBlank(currentRule.getCsId())) {
checkDatasourceInContextService(ruleDataSource, mappingCol, clusterName, userName, nodeName, currentRule.getCsId());
}
}
if (CollectionUtils.isNotEmpty(datasourceSizeList) && currentRule.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())) {
double left = datasourceSizeList.get(0);
double right = datasourceSizeList.get(1);
LOGGER.info("Current multi source rule left table size number[{}], right table size number[{}]", left, right);
if (left == 0 && right == 0) {
throw new BothNullDatasourceException("{&BOTH_SIDE_ARE_NULL}");
} else if (left == 0) {
throw new LeftNullDatasourceException("{&ONE_SIDE_ARE_NULL}");
} else if (right == 0) {
throw new RightNullDatasourceException("{&ONE_SIDE_ARE_NULL}");
}
}
}
use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method submitRulesWithDynamicPartition.
private void submitRulesWithDynamicPartition(List<ApplicationSubmitRequest> applicationSubmitRequests, Long projectId, Long ruleGroupId, List<Rule> rules, List<Long> ruleIds, String executionUser, boolean dynamicPartition, String clusterName, StringBuffer partition, String dynamicPartitionPrefix) throws UnExpectedRequestException, NoPartitionException {
if (dynamicPartition) {
for (Rule rule : rules) {
RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
if (ruleDataSource == null) {
throw new UnExpectedRequestException("Rule datasource has been broken.");
}
if (RuleTypeEnum.CUSTOM_RULE.getCode().equals(rule.getRuleType())) {
throw new UnExpectedRequestException(RuleTypeEnum.CUSTOM_RULE.getMessage() + " {&IS_NOT_SUPPORT}");
}
List<Map> currentPartitionMap;
try {
TableStatisticsInfo tableStatisticsInfo = metaDataClient.getTableStatisticsInfo(StringUtils.isBlank(clusterName) ? ruleDataSource.getClusterName() : clusterName, ruleDataSource.getDbName(), ruleDataSource.getTableName(), executionUser);
currentPartitionMap = tableStatisticsInfo.getPartitions();
if (StringUtils.isNotEmpty(dynamicPartitionPrefix)) {
metaDataClient.getPartitionStatisticsInfo(StringUtils.isBlank(clusterName) ? ruleDataSource.getClusterName() : clusterName, ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(dynamicPartitionPrefix), executionUser);
}
} catch (MetaDataAcquireFailedException e) {
LOGGER.error("Dynamic submit failed.", e);
throw new NoPartitionException(ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + "{&HAS_NO_PARTITIONS_TO_BE_EXECUTE}");
}
List<String> partitionList = new ArrayList<>();
if (CollectionUtils.isNotEmpty(currentPartitionMap)) {
ruleIds.remove(rule.getId());
getPartitionListAndSubmit(currentPartitionMap, partitionList, rule, partition, filterToPartitionPath(dynamicPartitionPrefix), applicationSubmitRequests, projectId, ruleGroupId);
} else {
throw new NoPartitionException(ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + "{&HAS_NO_PARTITIONS_TO_BE_EXECUTE}");
}
}
}
}
use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method dataSourceExecution.
@Override
public GeneralResponse<?> dataSourceExecution(DataSourceExecutionRequest request) throws UnExpectedRequestException, PermissionDeniedRequestException {
LOGGER.info("Execute application by datasource. cluster: {}, database: {}, table: {}", request.getCluster(), request.getDatabase(), request.getTable());
// Check Arguments.
DataSourceExecutionRequest.checkRequest(request);
String loginUser = getLoginUser(httpServletRequest, request.getCreateUser(), request.getAsync());
// Find all rule datasources by user.
List<RuleDataSource> ruleDataSources = new ArrayList<>();
ruleDataSources.addAll(ruleDataSourceDao.findDatasourcesByUser(loginUser, request.getCluster(), request.getDatabase(), request.getTable()));
List<Rule> rules = ruleDataSources.stream().filter(r -> r.getClusterName().equals(request.getCluster()) && r.getDbName().equals(request.getDatabase()) && r.getTableName().equals(request.getTable())).map(RuleDataSource::getRule).distinct().filter(rule -> {
if (!request.getCrossTable()) {
return rule.getRuleDataSources().size() == 1;
}
return true;
}).collect(Collectors.toList());
if (CollectionUtils.isEmpty(rules)) {
throw new UnExpectedRequestException("{&NO_RULE_CAN_BE_EXECUTED}");
}
checkPermissionCreateUserProxyExecuteUser(request.getCreateUser(), request.getExecutionUser());
ApplicationProjectResponse applicationProjectResponse = new ApplicationProjectResponse();
List<ApplicationSubmitRequest> applicationSubmitRequests = new ArrayList<>();
StringBuffer partition = new StringBuffer();
StringBuffer runDate = new StringBuffer();
Map<String, String> execParamMap = new HashMap<>(5);
parseExecParams(partition, runDate, request.getExecutionParam(), execParamMap);
List<Project> projects = rules.stream().map(Rule::getProject).distinct().collect(Collectors.toList());
for (Project projectInDb : projects) {
// Check permissions of project
List<Integer> permissions = new ArrayList<>();
permissions.add(ProjectUserPermissionEnum.OPERATOR.getCode());
projectService.checkProjectPermission(projectInDb, loginUser, permissions);
List<Rule> currentRules = rules.stream().filter(rule -> rule.getProject().getId().equals(projectInDb.getId())).collect(Collectors.toList());
List<RuleGroup> currentRuleGroups = currentRules.stream().map(Rule::getRuleGroup).distinct().collect(Collectors.toList());
for (RuleGroup ruleGroup : currentRuleGroups) {
List<Rule> currentRulesOfGroup = currentRules.stream().filter(rule -> rule.getRuleGroup().getRuleGroupName().equals(ruleGroup.getRuleGroupName())).distinct().collect(Collectors.toList());
List<Long> currentRuleIds = currentRulesOfGroup.stream().map(Rule::getId).distinct().collect(Collectors.toList());
LOGGER.info("Succeed to find current rules of one group with datasource. rule_id: {}", currentRuleIds);
// Dynamic partition.
try {
submitRulesWithDynamicPartition(applicationSubmitRequests, projectInDb.getId(), ruleGroup.getId(), rules, currentRuleIds, request.getExecutionUser(), request.getDyNamicPartition(), request.getClusterName(), partition, request.getDyNamicPartitionPrefix());
} catch (ResourceAccessException e) {
// Record submit failed applicatoin.
generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
} catch (NoPartitionException e) {
// Record submit failed applicatoin.
generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
} catch (Exception e) {
// Record submit failed applicatoin.
generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.UNKNOWN_ERROR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
}
if (CollectionUtils.isNotEmpty(currentRuleIds)) {
applicationSubmitRequests.add(new ApplicationSubmitRequest(request.getJobId(), projectInDb.getId(), ruleGroup.getId(), currentRuleIds, partition));
}
}
projectEventService.record(projectInDb.getId(), loginUser, "submit datasource execution", "rule name[" + Arrays.toString(rules.stream().filter(rule -> rule.getProject().getId().equals(projectInDb.getId())).toArray()) + "].", EventTypeEnum.SUBMIT_PROJECT.getCode());
}
GeneralResponse<ApplicationTaskSimpleResponse> generalResponse;
for (ApplicationSubmitRequest applicationSubmitRequest : applicationSubmitRequests) {
if (StringUtils.isNotBlank(request.getJobId())) {
applicationSubmitRequest.setJobId(request.getJobId());
}
generalResponse = (GeneralResponse<ApplicationTaskSimpleResponse>) outerExecutionService.submitRules(applicationSubmitRequest.getRuleIds(), applicationSubmitRequest.getPartition(), loginUser, request.getExecutionUser(), DEFAULT_NODE_NAME, applicationSubmitRequest.getProjectId(), applicationSubmitRequest.getRuleGroupId(), request.getStartupParamName(), request.getClusterName(), request.getSetFlag(), execParamMap, request.getExecutionParam(), runDate, InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode());
applicationProjectResponse.getApplicationTaskSimpleResponses().add(generalResponse.getData());
}
return new GeneralResponse<>("200", "{&SUCCEED_TO_DISPATCH_TASK}", applicationProjectResponse);
}
Aggregations