use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method submitRules.
/**
* Generate jobs by rules and submit jobs to linkis.
*
* @param ruleIds
* @param partition
* @param createUser
* @param executionUser
* @param nodeName
* @param projectId
* @param ruleGroupId
* @param execParams
* @param runDate
* @param invokeCode
* @return
* @throws UnExpectedRequestException
*/
@Override
@Transactional(propagation = Propagation.REQUIRES_NEW, rollbackFor = { RuntimeException.class, UnExpectedRequestException.class })
public GeneralResponse<?> submitRules(List<Long> ruleIds, StringBuffer partition, String createUser, String executionUser, String nodeName, Long projectId, Long ruleGroupId, String startupParam, String clusterName, String setFlag, Map<String, String> execParams, String execParamStr, StringBuffer runDate, Integer invokeCode) {
// Get rule eneity.
List<Rule> rules = ruleDao.findByIds(ruleIds);
// Init application basic info.
Date date = new Date();
Application newApplication = outerExecutionService.generateApplicationInfo(createUser, executionUser, date, invokeCode);
newApplication.setProjectId(projectId);
newApplication.setRuleGroupId(ruleGroupId);
newApplication.setPartition(partition.toString());
newApplication.setClusterName(clusterName);
newApplication.setStartupParam(startupParam);
newApplication.setExecutionParam(execParamStr);
newApplication.setSetFlag(setFlag);
ApplicationTaskSimpleResponse response;
try {
response = outerExecutionService.commonExecution(rules, partition, executionUser, nodeName, startupParam, clusterName, setFlag, execParams, newApplication, date, runDate);
} catch (BothNullDatasourceException e) {
catchAndSolve(e, ApplicationCommentEnum.BOTH_NULL_ISSUES.getCode(), ApplicationStatusEnum.FINISHED.getCode(), rules, newApplication);
return new GeneralResponse<>("500", e.getMessage(), null);
} catch (LeftNullDatasourceException e) {
catchAndSolve(e, ApplicationCommentEnum.LEFT_NULL_DATA_ISSUES.getCode(), ApplicationStatusEnum.NOT_PASS.getCode(), rules, newApplication);
return new GeneralResponse<>("500", e.getMessage(), null);
} catch (RightNullDatasourceException e) {
catchAndSolve(e, ApplicationCommentEnum.RIGHT_NULL_DATA_ISSUES.getCode(), ApplicationStatusEnum.NOT_PASS.getCode(), rules, newApplication);
return new GeneralResponse<>("500", e.getMessage(), null);
} catch (MetaDataAcquireFailedException e) {
catchAndSolve(e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
return new GeneralResponse<>("500", "{&THE_CHECK_FIELD_HAS_BEEN_MODIFIED}", null);
} catch (DataSourceMoveException e) {
catchAndSolve(e, ApplicationCommentEnum.PERMISSION_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
return new GeneralResponse<>("500", e.getMessage(), null);
} catch (DataSourceOverSizeException e) {
catchAndSolve(e, ApplicationCommentEnum.PERMISSION_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
return new GeneralResponse<>("500", e.getMessage(), null);
} catch (ParseException e) {
catchAndSolve(e, ApplicationCommentEnum.GRAMMAR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
LOGGER.error(e.getMessage(), e);
return new GeneralResponse<>("500", "{&PARSE_SQL_FAILED_PLEASE_CHECKOUT_YOUR_CUSTOM_SQL}", null);
} catch (Exception e) {
catchAndSolve(e, ApplicationCommentEnum.UNKNOWN_ERROR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
return new GeneralResponse<>("500", e.getMessage(), null);
}
LOGGER.info("Succeed to dispatch task. response: {}", response);
return new GeneralResponse<>("200", "{&SUCCEED_TO_DISPATCH_TASK}", response);
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method dataSourceExecution.
@Override
public GeneralResponse<?> dataSourceExecution(DataSourceExecutionRequest request) throws UnExpectedRequestException, PermissionDeniedRequestException {
LOGGER.info("Execute application by datasource. cluster: {}, database: {}, table: {}", request.getCluster(), request.getDatabase(), request.getTable());
// Check Arguments.
DataSourceExecutionRequest.checkRequest(request);
String loginUser = getLoginUser(httpServletRequest, request.getCreateUser(), request.getAsync());
// Find all rule datasources by user.
List<RuleDataSource> ruleDataSources = new ArrayList<>();
ruleDataSources.addAll(ruleDataSourceDao.findDatasourcesByUser(loginUser, request.getCluster(), request.getDatabase(), request.getTable()));
List<Rule> rules = ruleDataSources.stream().filter(r -> r.getClusterName().equals(request.getCluster()) && r.getDbName().equals(request.getDatabase()) && r.getTableName().equals(request.getTable())).map(RuleDataSource::getRule).distinct().filter(rule -> {
if (!request.getCrossTable()) {
return rule.getRuleDataSources().size() == 1;
}
return true;
}).collect(Collectors.toList());
if (CollectionUtils.isEmpty(rules)) {
throw new UnExpectedRequestException("{&NO_RULE_CAN_BE_EXECUTED}");
}
checkPermissionCreateUserProxyExecuteUser(request.getCreateUser(), request.getExecutionUser());
ApplicationProjectResponse applicationProjectResponse = new ApplicationProjectResponse();
List<ApplicationSubmitRequest> applicationSubmitRequests = new ArrayList<>();
StringBuffer partition = new StringBuffer();
StringBuffer runDate = new StringBuffer();
Map<String, String> execParamMap = new HashMap<>(5);
parseExecParams(partition, runDate, request.getExecutionParam(), execParamMap);
List<Project> projects = rules.stream().map(Rule::getProject).distinct().collect(Collectors.toList());
for (Project projectInDb : projects) {
// Check permissions of project
List<Integer> permissions = new ArrayList<>();
permissions.add(ProjectUserPermissionEnum.OPERATOR.getCode());
projectService.checkProjectPermission(projectInDb, loginUser, permissions);
List<Rule> currentRules = rules.stream().filter(rule -> rule.getProject().getId().equals(projectInDb.getId())).collect(Collectors.toList());
List<RuleGroup> currentRuleGroups = currentRules.stream().map(Rule::getRuleGroup).distinct().collect(Collectors.toList());
for (RuleGroup ruleGroup : currentRuleGroups) {
List<Rule> currentRulesOfGroup = currentRules.stream().filter(rule -> rule.getRuleGroup().getRuleGroupName().equals(ruleGroup.getRuleGroupName())).distinct().collect(Collectors.toList());
List<Long> currentRuleIds = currentRulesOfGroup.stream().map(Rule::getId).distinct().collect(Collectors.toList());
LOGGER.info("Succeed to find current rules of one group with datasource. rule_id: {}", currentRuleIds);
// Dynamic partition.
try {
submitRulesWithDynamicPartition(applicationSubmitRequests, projectInDb.getId(), ruleGroup.getId(), rules, currentRuleIds, request.getExecutionUser(), request.getDyNamicPartition(), request.getClusterName(), partition, request.getDyNamicPartitionPrefix());
} catch (ResourceAccessException e) {
// Record submit failed applicatoin.
generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
} catch (NoPartitionException e) {
// Record submit failed applicatoin.
generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
} catch (Exception e) {
// Record submit failed applicatoin.
generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.UNKNOWN_ERROR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
}
if (CollectionUtils.isNotEmpty(currentRuleIds)) {
applicationSubmitRequests.add(new ApplicationSubmitRequest(request.getJobId(), projectInDb.getId(), ruleGroup.getId(), currentRuleIds, partition));
}
}
projectEventService.record(projectInDb.getId(), loginUser, "submit datasource execution", "rule name[" + Arrays.toString(rules.stream().filter(rule -> rule.getProject().getId().equals(projectInDb.getId())).toArray()) + "].", EventTypeEnum.SUBMIT_PROJECT.getCode());
}
GeneralResponse<ApplicationTaskSimpleResponse> generalResponse;
for (ApplicationSubmitRequest applicationSubmitRequest : applicationSubmitRequests) {
if (StringUtils.isNotBlank(request.getJobId())) {
applicationSubmitRequest.setJobId(request.getJobId());
}
generalResponse = (GeneralResponse<ApplicationTaskSimpleResponse>) outerExecutionService.submitRules(applicationSubmitRequest.getRuleIds(), applicationSubmitRequest.getPartition(), loginUser, request.getExecutionUser(), DEFAULT_NODE_NAME, applicationSubmitRequest.getProjectId(), applicationSubmitRequest.getRuleGroupId(), request.getStartupParamName(), request.getClusterName(), request.getSetFlag(), execParamMap, request.getExecutionParam(), runDate, InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode());
applicationProjectResponse.getApplicationTaskSimpleResponses().add(generalResponse.getData());
}
return new GeneralResponse<>("200", "{&SUCCEED_TO_DISPATCH_TASK}", applicationProjectResponse);
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method ruleListExecution.
@Override
public GeneralResponse<?> ruleListExecution(RuleListExecutionRequest request, Integer invokeCode) throws UnExpectedRequestException, PermissionDeniedRequestException {
LOGGER.info("Execute application by rules. rule_ids: {}", request.getRuleList());
// Check Arguments.
RuleListExecutionRequest.checkRequest(request);
String loginUser = getLoginUser(httpServletRequest, request.getCreateUser(), request.getAsync());
// Check the existence of rule.
List<Rule> rules = ruleDao.findByIds(request.getRuleList());
if (CollectionUtils.isEmpty(rules)) {
throw new UnExpectedRequestException("{&NO_RULE_CAN_BE_EXECUTED}");
}
List<Long> ruleIds = rules.stream().map(Rule::getId).distinct().collect(Collectors.toList());
// Check rule exists on the request.
List<Long> notExistRules = request.getRuleList().stream().filter(ruleId -> !ruleIds.contains(ruleId)).collect(Collectors.toList());
if (!notExistRules.isEmpty()) {
throw new UnExpectedRequestException("The ids of rule: " + notExistRules.toString() + " {&DOES_NOT_EXIST}");
}
LOGGER.info("Succeed to find all rules.");
checkPermissionCreateUserProxyExecuteUser(request.getCreateUser(), request.getExecutionUser());
StringBuffer runDate = new StringBuffer();
StringBuffer partition = new StringBuffer();
Map<String, String> execParamMap = new HashMap<>(5);
parseExecParams(partition, runDate, request.getExecutionParam(), execParamMap);
ApplicationProjectResponse applicationProjectResponse = new ApplicationProjectResponse();
List<ApplicationSubmitRequest> applicationSubmitRequests = new ArrayList<>(rules.size());
List<Project> projects = rules.stream().map(Rule::getProject).distinct().collect(Collectors.toList());
GeneralResponse<ApplicationTaskSimpleResponse> generalResponse;
// Check permissions of projects.
for (Project projectInDb : projects) {
List<Integer> permissions = new ArrayList<>();
permissions.add(ProjectUserPermissionEnum.OPERATOR.getCode());
projectService.checkProjectPermission(projectInDb, loginUser, permissions);
List<Rule> currentRules = rules.stream().filter(rule -> rule.getProject().getId().equals(projectInDb.getId())).collect(Collectors.toList());
List<RuleGroup> currentRuleGroups = currentRules.stream().map(Rule::getRuleGroup).distinct().collect(Collectors.toList());
for (RuleGroup ruleGroup : currentRuleGroups) {
List<Rule> currentRulesOfGroup = currentRules.stream().filter(rule -> rule.getRuleGroup().getRuleGroupName().equals(ruleGroup.getRuleGroupName())).distinct().collect(Collectors.toList());
List<Long> currentRuleIds = currentRulesOfGroup.stream().map(Rule::getId).distinct().collect(Collectors.toList());
// Dynamic partition.
try {
submitRulesWithDynamicPartition(applicationSubmitRequests, projectInDb.getId(), ruleGroup.getId(), currentRulesOfGroup, currentRuleIds, request.getExecutionUser(), request.getDyNamicPartition(), request.getClusterName(), partition, request.getDyNamicPartitionPrefix());
if (CollectionUtils.isEmpty(currentRuleIds)) {
continue;
}
} catch (ResourceAccessException e) {
// Record submit failed applicatoin.
generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
LOGGER.error("One group execution[id={}] of the rule list execution start failed!", ruleGroup.getId());
} catch (NoPartitionException e) {
// Record submit failed applicatoin.
generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
LOGGER.error("One group execution[id={}] of the rule list execution start failed!", ruleGroup.getId());
} catch (Exception e) {
// Record submit failed applicatoin.
generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
LOGGER.error("One group execution[id={}] of the rule list execution start failed!", ruleGroup.getId());
}
applicationSubmitRequests.add(new ApplicationSubmitRequest(projectInDb.getId(), ruleGroup.getId(), currentRuleIds, partition));
}
projectEventService.record(projectInDb.getId(), loginUser, "submit rule list execution", "rule name[" + Arrays.toString(currentRules.toArray()) + "].", EventTypeEnum.SUBMIT_PROJECT.getCode());
}
for (ApplicationSubmitRequest applicationSubmitRequest : applicationSubmitRequests) {
generalResponse = (GeneralResponse<ApplicationTaskSimpleResponse>) outerExecutionService.submitRules(applicationSubmitRequest.getRuleIds(), applicationSubmitRequest.getPartition(), loginUser, request.getExecutionUser(), DEFAULT_NODE_NAME, applicationSubmitRequest.getProjectId(), applicationSubmitRequest.getRuleGroupId(), request.getStartupParamName(), request.getClusterName(), request.getSetFlag(), execParamMap, request.getExecutionParam(), runDate, invokeCode);
applicationProjectResponse.getApplicationTaskSimpleResponses().add(generalResponse.getData());
}
return new GeneralResponse<>("200", "{&SUCCEED_TO_DISPATCH_TASK}", applicationProjectResponse);
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method catchAndSolve.
private void catchAndSolve(Exception e, Integer commentCode, Integer statusCode, List<Rule> rules, Application newApplication) {
LOGGER.error(e.getMessage(), e);
newApplication.setApplicationComment(commentCode);
newApplication.setStatus(statusCode);
// Record rules info and datasource info.
StringBuffer info = new StringBuffer();
newApplication.setFinishTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
for (Rule rule : rules) {
info.append("Rule[").append(rule.getName()).append("; ");
info.append("datasource: ").append(rule.getRuleDataSources().stream().filter(ruleDataSource -> StringUtils.isNotBlank(ruleDataSource.getDbName()) && StringUtils.isNotBlank(ruleDataSource.getTableName())).map(ruleDataSource -> ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + " ").collect(Collectors.joining()));
}
newApplication.setExceptionMessage(info.append("]\n").append(ExceptionUtils.getStackTrace(e)).toString());
if (CollectionUtils.isNotEmpty(rules)) {
Project currentProject = rules.iterator().next().getProject();
newApplication.setProjectName(StringUtils.isNotBlank(currentProject.getCnName()) ? currentProject.getCnName() : currentProject.getName());
newApplication.setRuleGroupName(rules.iterator().next().getRuleGroup().getRuleGroupName());
newApplication.setRuleDatesource(rules.stream().map(Rule::getRuleDataSources).flatMap(ruleDataSources -> ruleDataSources.stream()).filter(ruleDataSource -> StringUtils.isNotBlank(ruleDataSource.getDbName()) && StringUtils.isNotBlank(ruleDataSource.getTableName())).map(ruleDataSource -> ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + " ").collect(Collectors.joining()));
}
applicationDao.saveApplication(newApplication);
LOGGER.info("Succeed to set application status to [{}], application_id: [{}]", newApplication.getStatus(), newApplication.getId());
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method generateApplicationInfo.
@Override
public Application generateApplicationInfo(String createUser, String executionUser, Date date, Integer invokeCode) {
String nonce = UuidGenerator.generateRandom(6);
String applicationId = generateTaskId(date, nonce);
LOGGER.info("Succeed to generate application_id: {}", applicationId);
String submitTime = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(date);
Application application = new Application();
application.setId(applicationId);
application.setSubmitTime(submitTime);
application.setInvokeType(invokeCode);
application.setCreateUser(createUser);
application.setExecuteUser(executionUser);
return applicationDao.saveApplication(application);
}
Aggregations