use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class ExecutionManagerImpl method submitApplication.
/**
* Submit job to linkis
*/
@Override
public List<TaskSubmitResult> submitApplication(List<Rule> rules, String nodeName, String createTime, String user, String database, StringBuffer partition, Date date, Application application, String cluster, String startupParam, String setFlag, Map<String, String> execParams, StringBuffer runDate, Map<Long, Map> dataSourceMysqlConnect) throws ArgumentException, TaskTypeException, ConvertException, DataQualityTaskException, RuleVariableNotSupportException, RuleVariableNotFoundException, JobSubmitException, ClusterInfoNotConfigException, IOException, UnExpectedRequestException, MetaDataAcquireFailedException {
String csId = rules.iterator().next().getCsId();
// Check if cluster supported
LOGGER.info("Start to collect rule to clusters");
Map<String, List<Rule>> clusterNameMap = getRuleCluster(rules);
LOGGER.info("Succeed to classify rules by cluster, cluster map: {}", clusterNameMap);
if (StringUtils.isNotBlank(cluster)) {
LOGGER.info("When pick up a cluster, these datasources of rules must be from one cluster. Now start to put into the specify cluster.\n");
putAllRulesIntoSpecifyCluster(clusterNameMap, cluster);
LOGGER.info("Success to put into the specify cluster.\n");
}
List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
for (String clusterName : clusterNameMap.keySet()) {
List<Rule> clusterRules = clusterNameMap.get(clusterName);
if (StringUtils.isNotBlank(cluster)) {
clusterName = cluster;
}
ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
LOGGER.info("Start to check cluster config.");
if (clusterInfo == null) {
throw new ClusterInfoNotConfigException(clusterName + " {&DOES_NOT_EXIST}");
}
LOGGER.info("Succeed to pass the check of cluster config. All cluster of rules are configured");
// Divide rule into tasks
List<DataQualityTask> tasks = TaskDividerFactory.getDivider().divide(clusterRules, application.getId(), createTime, partition.toString(), date, database, user, taskExecuteLimitConfig.getTaskExecuteRuleSize());
LOGGER.info("Succeed to divide application into tasks. result: {}", tasks);
// Save divided tasks
saveDividedTask(tasks, clusterInfo, rules, application, createTime);
// Convert tasks into job
List<DataQualityJob> jobList = new ArrayList<>();
for (DataQualityTask task : tasks) {
DataQualityJob job = templateConverterFactory.getConverter(task).convert(task, date, setFlag, execParams, runDate.toString(), clusterInfo.getClusterType(), dataSourceMysqlConnect);
job.setUser(task.getUser());
jobList.add(job);
List<Long> ruleIdList = task.getRuleTaskDetails().stream().map(r -> r.getRule().getId()).collect(Collectors.toList());
LOGGER.info("Succeed to convert rule_id: {} into code. code: {}", ruleIdList, job.getJobCode());
}
LOGGER.info("Succeed to convert all template into codes. codes: {}", jobList);
// Submit job to linkis
List<JobSubmitResult> submitResults = new ArrayList<>();
for (DataQualityJob job : jobList) {
String code = String.join("\n", job.getJobCode());
String proxy = job.getUser();
Long taskId = job.getTaskId();
// Compatible with new and old submission interfaces.
JobSubmitResult result = null;
boolean engineReUse = false;
if (StringUtils.isNotBlank(startupParam)) {
String[] startupParams = startupParam.split(SpecCharEnum.DIVIDER.getValue());
for (String param : startupParams) {
if (StringUtils.isEmpty(param)) {
continue;
}
String[] paramStrs = param.split("=");
if (paramStrs.length < 2) {
continue;
}
String key = paramStrs[0];
String value = paramStrs[1];
if ("engine_reuse".equals(key)) {
if ("true".equals(value)) {
engineReUse = true;
startupParam = startupParam.replace("engine_reuse=true", "");
} else {
engineReUse = false;
startupParam = startupParam.replace("engine_reuse=false", "");
}
break;
}
}
}
if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
result = abstractJobSubmitter.submitJobNew(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam(), engineReUse);
} else {
result = abstractJobSubmitter.submitJob(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam());
}
if (result != null) {
submitResults.add(result);
} else {
Task taskInDb = taskDao.findById(taskId);
taskInDb.setStatus(TaskStatusEnum.TASK_NOT_EXIST.getCode());
taskDao.save(taskInDb);
taskSubmitResults.add(new TaskSubmitResult(application.getId(), null, clusterInfo.getClusterName()));
}
}
// Rewrite task remote ID.
rewriteTaskRemoteInfo(submitResults, taskSubmitResults, application.getId(), clusterInfo.getClusterName());
}
return taskSubmitResults;
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method getApplicationStatus.
@Override
public GeneralResponse<?> getApplicationStatus(String applicationId) throws UnExpectedRequestException {
// Find application by applicationId
Application application = applicationDao.findById(applicationId);
if (application == null) {
throw new UnExpectedRequestException("Application_id {&DOES_NOT_EXIST}");
}
LOGGER.info("Succeed to find application. application: {}", application);
List<Task> tasks = taskDao.findByApplication(application);
ApplicationTaskResponse response = new ApplicationTaskResponse(application, tasks);
LOGGER.info("Succeed to get application status. response: {}", response);
return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATION_STATUS}", response);
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method getApplicationDynamicStatus.
@Override
public GeneralResponse<?> getApplicationDynamicStatus(String applicationId) throws UnExpectedRequestException {
// Find application by applicationId
Application application = applicationDao.findById(applicationId);
if (application == null) {
throw new UnExpectedRequestException("Application_id {&DOES_NOT_EXIST}");
}
LOGGER.info("Succeed to find application. application: {}", application);
return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATION_STATUS}", new ApplicationTaskResponse(application));
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method killApplication.
@Override
public GeneralResponse<?> killApplication(String applicationId, String executionUser) throws JobKillException, UnExpectedRequestException, ClusterInfoNotConfigException, PermissionDeniedRequestException {
LOGGER.info("Qualitis appjoint user: {}", executionUser);
Application applicationInDb = applicationDao.findById(applicationId);
if (applicationInDb == null) {
throw new UnExpectedRequestException(String.format("Application ID: %s {&DOES_NOT_EXIST}", applicationId));
}
Integer status = applicationInDb.getStatus();
if (status.equals(ApplicationStatusEnum.SUBMITTED.getCode()) || status.equals(ApplicationStatusEnum.RUNNING.getCode()) || status.equals(ApplicationStatusEnum.SUCCESSFUL_CREATE_APPLICATION.getCode())) {
String realExecutionUser = applicationInDb.getExecuteUser();
checkPermissionCreateUserProxyExecuteUser(executionUser, realExecutionUser);
return executionManager.killApplication(applicationInDb, realExecutionUser);
} else {
return new GeneralResponse<>("400", "{&APPLICATION_IS_FINISHED}", null);
}
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method getApplicationResult.
@Override
public GeneralResponse<?> getApplicationResult(String applicationId) throws UnExpectedRequestException {
// Find application by applicationId
Application applicationInDb = applicationDao.findById(applicationId);
if (applicationInDb == null) {
throw new UnExpectedRequestException(String.format("ApplicationId: %s {&DOES_NOT_EXIST}", applicationId));
}
Integer passNum = 0;
Integer failedNum = 0;
Integer notPassNum = 0;
StringBuilder passMessage = new StringBuilder();
StringBuilder failedMessage = new StringBuilder();
StringBuilder notPassMessage = new StringBuilder();
passMessage.append("The rules following below are pass(以下规则已通过校验): [");
failedMessage.append("The rules following below are failed(以下规则已失败): [");
notPassMessage.append("The rules following below are failed(以下规则未通过校验): [");
// Find all tasks by application
List<Task> tasks = taskDao.findByApplication(applicationInDb);
// Find task rule simple in all tasks
for (Task task : tasks) {
// Add failed num, pass num if task status equals to Succeed, Failed
if (task.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
failedNum += task.getTaskRuleSimples().size();
generateFailedMessage(task, failedMessage);
} else if (task.getStatus().equals(TaskStatusEnum.PASS_CHECKOUT.getCode())) {
passNum += task.getTaskRuleSimples().size();
generatePassMessage(task, passMessage);
} else if (task.getStatus().equals(TaskStatusEnum.FAIL_CHECKOUT.getCode())) {
// Find not pass task if task status equals to failed_checkout
for (TaskRuleSimple taskRuleSimple : task.getTaskRuleSimples()) {
Boolean isPass = true;
for (TaskRuleAlarmConfig taskRuleAlarmConfig : taskRuleSimple.getTaskRuleAlarmConfigList()) {
if (taskRuleAlarmConfig.getStatus().equals(AlarmConfigStatusEnum.NOT_PASS.getCode())) {
isPass = false;
break;
}
}
if (isPass) {
passNum++;
passMessage.append(taskRuleSimple.getRuleName()).append(", ");
} else {
notPassNum++;
notPassMessage.append(taskRuleSimple.getRuleName()).append(", ");
}
}
} else {
throw new UnExpectedRequestException(String.format("ApplicationId: %s {&DOES_NOT_FINISHED_YET}", applicationId));
}
}
passMessage.append("]").append(System.lineSeparator());
failedMessage.append("]").append(System.lineSeparator());
notPassMessage.append("]").append(System.lineSeparator());
String resultMessage = passMessage.toString() + failedMessage.toString() + notPassMessage.toString();
return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATION_RESULT}", new ApplicationResultResponse(passNum, failedNum, notPassNum, resultMessage));
}
Aggregations