use of com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException in project Qualitis by WeBankFinTech.
the class ExecutionManagerImpl method submitApplication.
/**
* Submit job to linkis
*/
@Override
public List<TaskSubmitResult> submitApplication(List<Rule> rules, String nodeName, String createTime, String user, String database, StringBuffer partition, Date date, Application application, String cluster, String startupParam, String setFlag, Map<String, String> execParams, StringBuffer runDate, Map<Long, Map> dataSourceMysqlConnect) throws ArgumentException, TaskTypeException, ConvertException, DataQualityTaskException, RuleVariableNotSupportException, RuleVariableNotFoundException, JobSubmitException, ClusterInfoNotConfigException, IOException, UnExpectedRequestException, MetaDataAcquireFailedException {
String csId = rules.iterator().next().getCsId();
// Check if cluster supported
LOGGER.info("Start to collect rule to clusters");
Map<String, List<Rule>> clusterNameMap = getRuleCluster(rules);
LOGGER.info("Succeed to classify rules by cluster, cluster map: {}", clusterNameMap);
if (StringUtils.isNotBlank(cluster)) {
LOGGER.info("When pick up a cluster, these datasources of rules must be from one cluster. Now start to put into the specify cluster.\n");
putAllRulesIntoSpecifyCluster(clusterNameMap, cluster);
LOGGER.info("Success to put into the specify cluster.\n");
}
List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
for (String clusterName : clusterNameMap.keySet()) {
List<Rule> clusterRules = clusterNameMap.get(clusterName);
if (StringUtils.isNotBlank(cluster)) {
clusterName = cluster;
}
ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
LOGGER.info("Start to check cluster config.");
if (clusterInfo == null) {
throw new ClusterInfoNotConfigException(clusterName + " {&DOES_NOT_EXIST}");
}
LOGGER.info("Succeed to pass the check of cluster config. All cluster of rules are configured");
// Divide rule into tasks
List<DataQualityTask> tasks = TaskDividerFactory.getDivider().divide(clusterRules, application.getId(), createTime, partition.toString(), date, database, user, taskExecuteLimitConfig.getTaskExecuteRuleSize());
LOGGER.info("Succeed to divide application into tasks. result: {}", tasks);
// Save divided tasks
saveDividedTask(tasks, clusterInfo, rules, application, createTime);
// Convert tasks into job
List<DataQualityJob> jobList = new ArrayList<>();
for (DataQualityTask task : tasks) {
DataQualityJob job = templateConverterFactory.getConverter(task).convert(task, date, setFlag, execParams, runDate.toString(), clusterInfo.getClusterType(), dataSourceMysqlConnect);
job.setUser(task.getUser());
jobList.add(job);
List<Long> ruleIdList = task.getRuleTaskDetails().stream().map(r -> r.getRule().getId()).collect(Collectors.toList());
LOGGER.info("Succeed to convert rule_id: {} into code. code: {}", ruleIdList, job.getJobCode());
}
LOGGER.info("Succeed to convert all template into codes. codes: {}", jobList);
// Submit job to linkis
List<JobSubmitResult> submitResults = new ArrayList<>();
for (DataQualityJob job : jobList) {
String code = String.join("\n", job.getJobCode());
String proxy = job.getUser();
Long taskId = job.getTaskId();
// Compatible with new and old submission interfaces.
JobSubmitResult result = null;
boolean engineReUse = false;
if (StringUtils.isNotBlank(startupParam)) {
String[] startupParams = startupParam.split(SpecCharEnum.DIVIDER.getValue());
for (String param : startupParams) {
if (StringUtils.isEmpty(param)) {
continue;
}
String[] paramStrs = param.split("=");
if (paramStrs.length < 2) {
continue;
}
String key = paramStrs[0];
String value = paramStrs[1];
if ("engine_reuse".equals(key)) {
if ("true".equals(value)) {
engineReUse = true;
startupParam = startupParam.replace("engine_reuse=true", "");
} else {
engineReUse = false;
startupParam = startupParam.replace("engine_reuse=false", "");
}
break;
}
}
}
if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
result = abstractJobSubmitter.submitJobNew(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam(), engineReUse);
} else {
result = abstractJobSubmitter.submitJob(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam());
}
if (result != null) {
submitResults.add(result);
} else {
Task taskInDb = taskDao.findById(taskId);
taskInDb.setStatus(TaskStatusEnum.TASK_NOT_EXIST.getCode());
taskDao.save(taskInDb);
taskSubmitResults.add(new TaskSubmitResult(application.getId(), null, clusterInfo.getClusterName()));
}
}
// Rewrite task remote ID.
rewriteTaskRemoteInfo(submitResults, taskSubmitResults, application.getId(), clusterInfo.getClusterName());
}
return taskSubmitResults;
}
use of com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException in project Qualitis by WeBankFinTech.
the class ExecutionManagerImpl method killApplication.
@Override
public GeneralResponse<?> killApplication(Application applicationInDb, String user) throws JobKillException, UnExpectedRequestException, ClusterInfoNotConfigException {
List<Task> tasks = taskDao.findByApplication(applicationInDb);
List<JobKillResult> results = new ArrayList<>();
if (tasks == null || tasks.isEmpty()) {
throw new UnExpectedRequestException("Sub tasks {&CAN_NOT_BE_NULL_OR_EMPTY}");
}
for (Task task : tasks) {
ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(task.getClusterName());
if (clusterInfo == null) {
throw new ClusterInfoNotConfigException("Failed to find cluster id: " + task.getClusterName() + " configuration");
}
results.add(abstractJobSubmitter.killJob(user, clusterInfo.getClusterName(), task));
task.setStatus(TaskStatusEnum.CANCELLED.getCode());
task.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
taskDao.save(task);
}
return new GeneralResponse<>("200", "{&SUCCESS_TO_KILL_TASK}", results.size());
}
use of com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException in project Qualitis by WeBankFinTech.
the class LinkisJobSubmitter method submitJobNew.
@Override
public JobSubmitResult submitJobNew(String code, String engineName, String user, String remoteAddress, String clusterName, Long taskId, String csId, String nodeName, String startupParam, boolean engineReUse) throws JobSubmitException, ClusterInfoNotConfigException {
String url = getPath(remoteAddress).path(linkisConfig.getSubmitJobNew()).toString();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.add("Token-User", user);
headers.add("Token-Code", getToken(clusterName));
Gson gson = new Gson();
Map<String, Object> map = new HashMap<>(4);
Map<String, String> executionContent = new HashMap<>(2);
executionContent.put("code", code);
executionContent.put("runType", "scala");
map.put("executionContent", executionContent);
map.put("executeApplicationName", engineName);
map.put("executeUser", user);
Map<String, Map> params = new HashMap<>(2);
Map<String, Map> configuration = new HashMap<>(2);
Map<String, String> runtime = new HashMap<>(2);
runtime.put("contextID", csId);
runtime.put("nodeName", nodeName);
configuration.put("runtime", runtime);
Map<String, Object> startup;
Map<String, String> labels = new HashMap<>(4);
if (!engineReUse) {
labels.put("executeOnce", "");
}
if (StringUtils.isNotBlank(startupParam)) {
String[] startupParams = startupParam.split(SpecCharEnum.DIVIDER.getValue());
startup = new HashMap<>(startupParams.length);
for (String param : startupParams) {
if (StringUtils.isBlank(param)) {
continue;
}
String[] paramStrs = param.split("=");
if (paramStrs.length < 2) {
continue;
}
String key = paramStrs[0];
String value = paramStrs[1];
Matcher matcher = NUBBER_PATTERN.matcher(value);
if (matcher.matches()) {
startup.put(key, Integer.parseInt(value));
} else {
startup.put(key, value);
}
}
configuration.put("startup", startup);
}
params.put("configuration", configuration);
map.put("params", params);
labels.put("engineType", linkisConfig.getEngineName() + "-" + linkisConfig.getEngineVersion());
labels.put("userCreator", user + "-" + linkisConfig.getAppName());
labels.put("codeLanguageType", "scala");
map.put("labels", labels);
HttpEntity<Object> entity = new HttpEntity<>(gson.toJson(map), headers);
LOGGER.info("Start to submit job to linkis 1.0. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.POST, entity);
Map<String, Object> response = null;
try {
response = restTemplate.postForObject(url, entity, Map.class);
} catch (Exception e) {
LOGGER.error(e.getMessage());
throw new JobSubmitException("{&FAILED_TO_SUBMIT_TO_LINKIS}. Exception: " + e.getMessage());
}
LOGGER.info("Succeed to submit job to linkis 1.0. response: {}", response);
if (!checkResponse(response)) {
String message = (String) response.get("message");
throw new JobSubmitException("{&FAILED_TO_SUBMIT_TO_LINKIS}. Exception: " + message);
}
Long jobId = ((Integer) ((Map<String, Object>) response.get("data")).get("taskID")).longValue();
String execId = (String) ((Map<String, Object>) response.get("data")).get("execID");
String status = "";
return new JobSubmitResult(taskId, status, clusterName, remoteAddress, jobId, execId);
}
use of com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method getTaskLog.
@Override
public GeneralResponse<?> getTaskLog(GetTaskLogRequest request) throws UnExpectedRequestException, PermissionDeniedRequestException {
// Check Arguments
GetTaskLogRequest.checkRequest(request);
Task task = taskDao.findByRemoteTaskIdAndClusterName(request.getTaskId(), request.getClusterId());
if (task == null) {
throw new UnExpectedRequestException("Task_id {&DOES_NOT_EXIST}");
}
ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(request.getClusterId());
if (clusterInfo == null) {
throw new UnExpectedRequestException("cluster : [" + request.getClusterId() + "] {&DOES_NOT_EXIST}");
}
String executeUser = task.getApplication().getExecuteUser();
// Check if user has permissions proxying execution user
checkPermissionCreateUserProxyExecuteUser(request.getCreateUser(), executeUser);
// Check if user has permissions to view this task
if (!request.getCreateUser().equals(task.getApplication().getCreateUser())) {
throw new UnExpectedRequestException(String.format("User: %s {&HAS_NO_PERMISSION_TO_ACCESS} task: %s", request.getCreateUser(), request.getTaskId()), 403);
}
LogResult logResult;
try {
logResult = monitorManager.getTaskPartialLog(request.getTaskId(), 0, executeUser, clusterInfo.getLinkisAddress(), clusterInfo.getClusterName());
} catch (LogPartialException | ClusterInfoNotConfigException e) {
throw new UnExpectedRequestException(e.getMessage());
}
LOGGER.info("Succeed to get log of the task. task_id: {}, cluster_id: {}", request.getTaskId(), request.getClusterId());
return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_TASK_LOG}", logResult.getLog());
}
use of com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException in project Qualitis by WeBankFinTech.
the class JobServiceImpl method getTaskLog.
@Override
public GeneralResponse<?> getTaskLog(Long taskId, String clusterName) throws UnExpectedRequestException {
Task task = taskDao.findById(taskId);
if (task == null) {
throw new UnExpectedRequestException("{&JOB_ID_DOES_NOT_EXIST}");
}
ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
if (clusterInfo == null) {
throw new UnExpectedRequestException("Cluster info {&DOES_NOT_EXIST}");
}
LogResult logResult;
String proxyUser = task.getTaskProxyUser();
try {
logResult = monitorManager.getTaskPartialLog(task.getTaskRemoteId(), 0, StringUtils.isNotBlank(proxyUser) ? proxyUser : task.getApplication().getExecuteUser(), clusterInfo.getLinkisAddress(), clusterName);
} catch (LogPartialException | ClusterInfoNotConfigException e) {
throw new UnExpectedRequestException(e.getMessage());
}
LOGGER.info("Succeed to get task log, task_id: {}, cluster_id: {}", taskId, clusterName);
return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_TASK_LOG}", logResult.getLog());
}
Aggregations