Examples with Task com.webank.wedatasphere.qualitis.entity.Task used on opensource projects

Example 1 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method submitApplication.

/**
 * Submit job to linkis
 */
@Override
public List<TaskSubmitResult> submitApplication(List<Rule> rules, String nodeName, String createTime, String user, String database, StringBuffer partition, Date date, Application application, String cluster, String startupParam, String setFlag, Map<String, String> execParams, StringBuffer runDate, Map<Long, Map> dataSourceMysqlConnect) throws ArgumentException, TaskTypeException, ConvertException, DataQualityTaskException, RuleVariableNotSupportException, RuleVariableNotFoundException, JobSubmitException, ClusterInfoNotConfigException, IOException, UnExpectedRequestException, MetaDataAcquireFailedException {
    String csId = rules.iterator().next().getCsId();
    // Check if cluster supported
    LOGGER.info("Start to collect rule to clusters");
    Map<String, List<Rule>> clusterNameMap = getRuleCluster(rules);
    LOGGER.info("Succeed to classify rules by cluster, cluster map: {}", clusterNameMap);
    if (StringUtils.isNotBlank(cluster)) {
        LOGGER.info("When pick up a cluster, these datasources of rules must be from one cluster. Now start to put into the specify cluster.\n");
        putAllRulesIntoSpecifyCluster(clusterNameMap, cluster);
        LOGGER.info("Success to put into the specify cluster.\n");
    }
    List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
    for (String clusterName : clusterNameMap.keySet()) {
        List<Rule> clusterRules = clusterNameMap.get(clusterName);
        if (StringUtils.isNotBlank(cluster)) {
            clusterName = cluster;
        }
        ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
        LOGGER.info("Start to check cluster config.");
        if (clusterInfo == null) {
            throw new ClusterInfoNotConfigException(clusterName + " {&DOES_NOT_EXIST}");
        }
        LOGGER.info("Succeed to pass the check of cluster config. All cluster of rules are configured");
        // Divide rule into tasks
        List<DataQualityTask> tasks = TaskDividerFactory.getDivider().divide(clusterRules, application.getId(), createTime, partition.toString(), date, database, user, taskExecuteLimitConfig.getTaskExecuteRuleSize());
        LOGGER.info("Succeed to divide application into tasks. result: {}", tasks);
        // Save divided tasks
        saveDividedTask(tasks, clusterInfo, rules, application, createTime);
        // Convert tasks into job
        List<DataQualityJob> jobList = new ArrayList<>();
        for (DataQualityTask task : tasks) {
            DataQualityJob job = templateConverterFactory.getConverter(task).convert(task, date, setFlag, execParams, runDate.toString(), clusterInfo.getClusterType(), dataSourceMysqlConnect);
            job.setUser(task.getUser());
            jobList.add(job);
            List<Long> ruleIdList = task.getRuleTaskDetails().stream().map(r -> r.getRule().getId()).collect(Collectors.toList());
            LOGGER.info("Succeed to convert rule_id: {} into code. code: {}", ruleIdList, job.getJobCode());
        }
        LOGGER.info("Succeed to convert all template into codes. codes: {}", jobList);
        // Submit job to linkis
        List<JobSubmitResult> submitResults = new ArrayList<>();
        for (DataQualityJob job : jobList) {
            String code = String.join("\n", job.getJobCode());
            String proxy = job.getUser();
            Long taskId = job.getTaskId();
            // Compatible with new and old submission interfaces.
            JobSubmitResult result = null;
            boolean engineReUse = false;
            if (StringUtils.isNotBlank(startupParam)) {
                String[] startupParams = startupParam.split(SpecCharEnum.DIVIDER.getValue());
                for (String param : startupParams) {
                    if (StringUtils.isEmpty(param)) {
                        continue;
                    }
                    String[] paramStrs = param.split("=");
                    if (paramStrs.length < 2) {
                        continue;
                    }
                    String key = paramStrs[0];
                    String value = paramStrs[1];
                    if ("engine_reuse".equals(key)) {
                        if ("true".equals(value)) {
                            engineReUse = true;
                            startupParam = startupParam.replace("engine_reuse=true", "");
                        } else {
                            engineReUse = false;
                            startupParam = startupParam.replace("engine_reuse=false", "");
                        }
                        break;
                    }
                }
            }
            if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
                result = abstractJobSubmitter.submitJobNew(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam(), engineReUse);
            } else {
                result = abstractJobSubmitter.submitJob(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam());
            }
            if (result != null) {
                submitResults.add(result);
            } else {
                Task taskInDb = taskDao.findById(taskId);
                taskInDb.setStatus(TaskStatusEnum.TASK_NOT_EXIST.getCode());
                taskDao.save(taskInDb);
                taskSubmitResults.add(new TaskSubmitResult(application.getId(), null, clusterInfo.getClusterName()));
            }
        }
        // Rewrite task remote ID.
        rewriteTaskRemoteInfo(submitResults, taskSubmitResults, application.getId(), clusterInfo.getClusterName());
    }
    return taskSubmitResults;
}

Example 2 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method killApplication.

@Override
public GeneralResponse<?> killApplication(Application applicationInDb, String user) throws JobKillException, UnExpectedRequestException, ClusterInfoNotConfigException {
    List<Task> tasks = taskDao.findByApplication(applicationInDb);
    List<JobKillResult> results = new ArrayList<>();
    if (tasks == null || tasks.isEmpty()) {
        throw new UnExpectedRequestException("Sub tasks {&CAN_NOT_BE_NULL_OR_EMPTY}");
    }
    for (Task task : tasks) {
        ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(task.getClusterName());
        if (clusterInfo == null) {
            throw new ClusterInfoNotConfigException("Failed to find cluster id: " + task.getClusterName() + " configuration");
        }
        results.add(abstractJobSubmitter.killJob(user, clusterInfo.getClusterName(), task));
        task.setStatus(TaskStatusEnum.CANCELLED.getCode());
        task.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
        taskDao.save(task);
    }
    return new GeneralResponse<>("200", "{&SUCCESS_TO_KILL_TASK}", results.size());
}

Example 3 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method executeFileRule.

@Override
public TaskSubmitResult executeFileRule(List<Rule> fileRules, String submitTime, Application application, String user, String clusterName, StringBuffer runDate) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    LOGGER.info("Start to execute file rule task and save check result.");
    Task taskInDb = taskDao.save(new Task(application, submitTime, TaskStatusEnum.SUBMITTED.getCode()));
    Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
    Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
    int totalRules = fileRules.size();
    int successRule = 0;
    for (Rule rule : fileRules) {
        if (rule.getAbortOnFailure() != null) {
            taskInDb.setAbortOnFailure(rule.getAbortOnFailure());
        }
        TaskRuleSimple taskRuleSimple = new TaskRuleSimple(rule, taskInDb, httpServletRequest.getHeader("Content-Language"));
        taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
        RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
        taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
        // Check rule datasource: 1) table 2) partition.
        if (StringUtils.isEmpty(ruleDataSource.getFilter())) {
            TableStatisticsInfo result;
            try {
                String proxyUser = ruleDataSource.getProxyUser();
                result = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
            } catch (RestClientException e) {
                LOGGER.error("Failed to get table statistics with linkis api.", e);
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            if (result == null) {
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            String fullSize = result.getTableSize();
            List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getTableFileCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
            successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
        } else {
            PartitionStatisticsInfo result;
            try {
                String proxyUser = ruleDataSource.getProxyUser();
                result = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(DateExprReplaceUtil.replaceFilter(new Date(), ruleDataSource.getFilter())), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
            } catch (RestClientException e) {
                LOGGER.error("Failed to get table statistics with linkis api.", e);
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            if (result == null) {
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            String fullSize = result.getPartitionSize();
            List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getPartitionChildCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
            successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
        }
        if (taskInDb.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
            break;
        }
    }
    taskInDb.setTaskDataSources(taskDataSources);
    taskInDb.setTaskRuleSimples(taskRuleSimples);
    // Update task status
    taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
    if (totalRules == successRule) {
        taskInDb.setStatus(TaskStatusEnum.PASS_CHECKOUT.getCode());
        taskInDb.setProgress(Double.parseDouble("1"));
    }
    taskDao.save(taskInDb);
    LOGGER.info("Finished to execute file rule task and save check result.");
    TaskSubmitResult taskSubmitResult = new TaskSubmitResult();
    taskSubmitResult.setApplicationId(application.getId());
    taskSubmitResult.setClusterName(clusterName);
    return taskSubmitResult;
}

Example 4 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method saveDividedTask.

private void saveDividedTask(List<DataQualityTask> dataQualityTasks, ClusterInfo clusterInfo, List<Rule> rules, Application application, String createTime) {
    for (DataQualityTask dataQualityTask : dataQualityTasks) {
        List<TaskRule> ruleList = getRule(rules, dataQualityTask);
        Task task = new Task(application, createTime, TaskStatusEnum.SUBMITTED.getCode(), Double.parseDouble("0"), clusterInfo.getClusterName(), clusterInfo.getLinkisAddress());
        Boolean abortOnFailure = false;
        for (Rule rule : rules) {
            if (rule.getAbortOnFailure()) {
                abortOnFailure = true;
                break;
            }
        }
        task.setAbortOnFailure(abortOnFailure);
        task.setTaskProxyUser(dataQualityTask.getUser());
        Task taskInDb = taskDao.save(task);
        LOGGER.info("Succeed to save task. task_id: {}", taskInDb.getId());
        saveJobRuleSimpleAndJobDataSource(ruleList, taskInDb);
        dataQualityTask.setTaskId(taskInDb.getId());
    }
}

Example 5 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method getApplicationStatus.

@Override
public GeneralResponse<?> getApplicationStatus(String applicationId) throws UnExpectedRequestException {
    // Find application by applicationId
    Application application = applicationDao.findById(applicationId);
    if (application == null) {
        throw new UnExpectedRequestException("Application_id {&DOES_NOT_EXIST}");
    }
    LOGGER.info("Succeed to find application. application: {}", application);
    List<Task> tasks = taskDao.findByApplication(application);
    ApplicationTaskResponse response = new ApplicationTaskResponse(application, tasks);
    LOGGER.info("Succeed to get application status. response: {}", response);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATION_STATUS}", response);
}