use of com.webank.wedatasphere.qualitis.bean.TaskSubmitResult in project Qualitis by WeBankFinTech.
the class ExecutionManagerImpl method submitApplication.
/**
* Submit job to linkis
*/
@Override
public List<TaskSubmitResult> submitApplication(List<Rule> rules, String nodeName, String createTime, String user, String database, StringBuffer partition, Date date, Application application, String cluster, String startupParam, String setFlag, Map<String, String> execParams, StringBuffer runDate, Map<Long, Map> dataSourceMysqlConnect) throws ArgumentException, TaskTypeException, ConvertException, DataQualityTaskException, RuleVariableNotSupportException, RuleVariableNotFoundException, JobSubmitException, ClusterInfoNotConfigException, IOException, UnExpectedRequestException, MetaDataAcquireFailedException {
String csId = rules.iterator().next().getCsId();
// Check if cluster supported
LOGGER.info("Start to collect rule to clusters");
Map<String, List<Rule>> clusterNameMap = getRuleCluster(rules);
LOGGER.info("Succeed to classify rules by cluster, cluster map: {}", clusterNameMap);
if (StringUtils.isNotBlank(cluster)) {
LOGGER.info("When pick up a cluster, these datasources of rules must be from one cluster. Now start to put into the specify cluster.\n");
putAllRulesIntoSpecifyCluster(clusterNameMap, cluster);
LOGGER.info("Success to put into the specify cluster.\n");
}
List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
for (String clusterName : clusterNameMap.keySet()) {
List<Rule> clusterRules = clusterNameMap.get(clusterName);
if (StringUtils.isNotBlank(cluster)) {
clusterName = cluster;
}
ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
LOGGER.info("Start to check cluster config.");
if (clusterInfo == null) {
throw new ClusterInfoNotConfigException(clusterName + " {&DOES_NOT_EXIST}");
}
LOGGER.info("Succeed to pass the check of cluster config. All cluster of rules are configured");
// Divide rule into tasks
List<DataQualityTask> tasks = TaskDividerFactory.getDivider().divide(clusterRules, application.getId(), createTime, partition.toString(), date, database, user, taskExecuteLimitConfig.getTaskExecuteRuleSize());
LOGGER.info("Succeed to divide application into tasks. result: {}", tasks);
// Save divided tasks
saveDividedTask(tasks, clusterInfo, rules, application, createTime);
// Convert tasks into job
List<DataQualityJob> jobList = new ArrayList<>();
for (DataQualityTask task : tasks) {
DataQualityJob job = templateConverterFactory.getConverter(task).convert(task, date, setFlag, execParams, runDate.toString(), clusterInfo.getClusterType(), dataSourceMysqlConnect);
job.setUser(task.getUser());
jobList.add(job);
List<Long> ruleIdList = task.getRuleTaskDetails().stream().map(r -> r.getRule().getId()).collect(Collectors.toList());
LOGGER.info("Succeed to convert rule_id: {} into code. code: {}", ruleIdList, job.getJobCode());
}
LOGGER.info("Succeed to convert all template into codes. codes: {}", jobList);
// Submit job to linkis
List<JobSubmitResult> submitResults = new ArrayList<>();
for (DataQualityJob job : jobList) {
String code = String.join("\n", job.getJobCode());
String proxy = job.getUser();
Long taskId = job.getTaskId();
// Compatible with new and old submission interfaces.
JobSubmitResult result = null;
boolean engineReUse = false;
if (StringUtils.isNotBlank(startupParam)) {
String[] startupParams = startupParam.split(SpecCharEnum.DIVIDER.getValue());
for (String param : startupParams) {
if (StringUtils.isEmpty(param)) {
continue;
}
String[] paramStrs = param.split("=");
if (paramStrs.length < 2) {
continue;
}
String key = paramStrs[0];
String value = paramStrs[1];
if ("engine_reuse".equals(key)) {
if ("true".equals(value)) {
engineReUse = true;
startupParam = startupParam.replace("engine_reuse=true", "");
} else {
engineReUse = false;
startupParam = startupParam.replace("engine_reuse=false", "");
}
break;
}
}
}
if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
result = abstractJobSubmitter.submitJobNew(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam(), engineReUse);
} else {
result = abstractJobSubmitter.submitJob(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam());
}
if (result != null) {
submitResults.add(result);
} else {
Task taskInDb = taskDao.findById(taskId);
taskInDb.setStatus(TaskStatusEnum.TASK_NOT_EXIST.getCode());
taskDao.save(taskInDb);
taskSubmitResults.add(new TaskSubmitResult(application.getId(), null, clusterInfo.getClusterName()));
}
}
// Rewrite task remote ID.
rewriteTaskRemoteInfo(submitResults, taskSubmitResults, application.getId(), clusterInfo.getClusterName());
}
return taskSubmitResults;
}
use of com.webank.wedatasphere.qualitis.bean.TaskSubmitResult in project Qualitis by WeBankFinTech.
the class ExecutionManagerImpl method executeFileRule.
@Override
public TaskSubmitResult executeFileRule(List<Rule> fileRules, String submitTime, Application application, String user, String clusterName, StringBuffer runDate) throws UnExpectedRequestException, MetaDataAcquireFailedException {
LOGGER.info("Start to execute file rule task and save check result.");
Task taskInDb = taskDao.save(new Task(application, submitTime, TaskStatusEnum.SUBMITTED.getCode()));
Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
int totalRules = fileRules.size();
int successRule = 0;
for (Rule rule : fileRules) {
if (rule.getAbortOnFailure() != null) {
taskInDb.setAbortOnFailure(rule.getAbortOnFailure());
}
TaskRuleSimple taskRuleSimple = new TaskRuleSimple(rule, taskInDb, httpServletRequest.getHeader("Content-Language"));
taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
// Check rule datasource: 1) table 2) partition.
if (StringUtils.isEmpty(ruleDataSource.getFilter())) {
TableStatisticsInfo result;
try {
String proxyUser = ruleDataSource.getProxyUser();
result = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
} catch (RestClientException e) {
LOGGER.error("Failed to get table statistics with linkis api.", e);
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
if (result == null) {
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
String fullSize = result.getTableSize();
List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getTableFileCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
} else {
PartitionStatisticsInfo result;
try {
String proxyUser = ruleDataSource.getProxyUser();
result = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(DateExprReplaceUtil.replaceFilter(new Date(), ruleDataSource.getFilter())), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
} catch (RestClientException e) {
LOGGER.error("Failed to get table statistics with linkis api.", e);
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
if (result == null) {
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
String fullSize = result.getPartitionSize();
List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getPartitionChildCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
}
if (taskInDb.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
break;
}
}
taskInDb.setTaskDataSources(taskDataSources);
taskInDb.setTaskRuleSimples(taskRuleSimples);
// Update task status
taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
if (totalRules == successRule) {
taskInDb.setStatus(TaskStatusEnum.PASS_CHECKOUT.getCode());
taskInDb.setProgress(Double.parseDouble("1"));
}
taskDao.save(taskInDb);
LOGGER.info("Finished to execute file rule task and save check result.");
TaskSubmitResult taskSubmitResult = new TaskSubmitResult();
taskSubmitResult.setApplicationId(application.getId());
taskSubmitResult.setClusterName(clusterName);
return taskSubmitResult;
}
use of com.webank.wedatasphere.qualitis.bean.TaskSubmitResult in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method commonExecution.
@Override
public ApplicationTaskSimpleResponse commonExecution(List<Rule> rules, StringBuffer partition, String executionUser, String nodeName, String startupParam, String clusterName, String setFlag, Map<String, String> execParams, Application newApplication, Date date, StringBuffer runDate) throws RuleVariableNotSupportException, JobSubmitException, RuleVariableNotFoundException, ArgumentException, ConvertException, DataQualityTaskException, TaskTypeException, ClusterInfoNotConfigException, SystemConfigException, UnExpectedRequestException, MetaDataAcquireFailedException, IOException, TaskNotExistException, SemanticException, ParseException, DataSourceMoveException, DataSourceOverSizeException, org.apache.hadoop.hive.ql.parse.ParseException, BothNullDatasourceException, RightNullDatasourceException, LeftNullDatasourceException, java.text.ParseException {
// current user
String userName = executionUser;
// Generate database name.
String database = generateDatabase(userName);
LOGGER.info("Succeed to generate database_name: {}", database);
// Save application
newApplication.setRuleSize(rules.size());
newApplication.setSavedDb(database);
Application saveApplication = applicationDao.saveApplication(newApplication);
List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
List<Rule> fileRules = new ArrayList<>();
Map<Long, Map> dataSourceMysqlConnect = new HashMap<>(2);
for (Iterator<Rule> iterator = rules.iterator(); iterator.hasNext(); ) {
Rule currentRule = iterator.next();
if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
// Replace with execution parameter and parse datasource to save.
customReSaveDateSource(currentRule, execParams, clusterName, date);
}
List<Map<String, String>> mappingCols = new ArrayList<>();
getMappingCols(currentRule, mappingCols);
// Check datasource before submit job.
try {
checkDatasource(currentRule, userName, partition, mappingCols, nodeName, clusterName, dataSourceMysqlConnect);
} catch (BothNullDatasourceException e) {
Task taskInDb = taskDao.save(new Task(newApplication, newApplication.getSubmitTime(), TaskStatusEnum.SUCCEED.getCode()));
taskInDb.setClusterName(clusterName);
TaskRuleSimple taskRuleSimple = new TaskRuleSimple(currentRule, taskInDb);
Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
}
taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
taskInDb.setTaskDataSources(taskDataSources);
taskInDb.setTaskRuleSimples(taskRuleSimples);
TaskResult taskResult = new TaskResult();
taskResult.setApplicationId(newApplication.getId());
taskResult.setCreateTime(newApplication.getSubmitTime());
taskResult.setRuleId(currentRule.getId());
taskResult.setResultType("Long");
taskResult.setValue(0 + "");
taskResultDao.saveTaskResult(taskResult);
taskDao.save(taskInDb);
iterator.remove();
if (!iterator.hasNext()) {
throw e;
}
continue;
}
if (currentRule.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())) {
fileRules.add(currentRule);
LOGGER.info("Succeed to find file rule. Rule: {}", currentRule.getId() + " " + currentRule.getName());
iterator.remove();
}
}
String submitTime = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(date);
// General task.
if (!rules.isEmpty()) {
taskSubmitResults.addAll(executionManager.submitApplication(rules, nodeName, submitTime, userName, database, partition, date, saveApplication, clusterName, startupParam, setFlag, execParams, runDate, dataSourceMysqlConnect));
}
// Execute file rule task and save task result.
if (!fileRules.isEmpty()) {
taskSubmitResults.add(executionManager.executeFileRule(fileRules, submitTime, saveApplication, userName, clusterName, runDate));
}
saveApplication.setTotalTaskNum(taskSubmitResults.size());
LOGGER.info("Succeed to submit application. result: {}", taskSubmitResults);
Application applicationInDb = applicationDao.saveApplication(saveApplication);
LOGGER.info("Succeed to save application. application: {}", applicationInDb);
return new ApplicationTaskSimpleResponse(taskSubmitResults);
}
use of com.webank.wedatasphere.qualitis.bean.TaskSubmitResult in project Qualitis by WeBankFinTech.
the class ExecutionManagerImpl method rewriteTaskRemoteInfo.
private void rewriteTaskRemoteInfo(List<JobSubmitResult> submitResults, List<TaskSubmitResult> taskSubmitResults, String id, String clusterName) {
for (JobSubmitResult jobSubmitResult : submitResults) {
Task taskInDb = taskDao.findById(jobSubmitResult.getTaskId());
taskInDb.setTaskRemoteId(jobSubmitResult.getTaskRemoteId());
taskInDb.setTaskExecId(jobSubmitResult.getTaskExecId());
taskDao.save(taskInDb);
taskSubmitResults.add(new TaskSubmitResult(id, jobSubmitResult.getTaskRemoteId(), clusterName));
}
}
Aggregations