Examples with Task com.webank.wedatasphere.qualitis.entity.Task used on opensource projects

Example 16 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method filterDataSourceApplication.

@Override
public GeneralResponse<?> filterDataSourceApplication(FilterDataSourceRequest request) throws UnExpectedRequestException {
    // Check arguments
    FilterDataSourceRequest.checkRequest(request);
    Integer page = request.getPage();
    Integer size = request.getSize();
    String clusterName = request.getClusterName();
    String databaseName = StringUtils.isEmpty(request.getDatabaseName()) ? "" : request.getDatabaseName();
    String tableName = StringUtils.isEmpty(request.getTableName()) ? "" : request.getTableName();
    Long userId = HttpUtils.getUserId(httpServletRequest);
    User user = userDao.findById(userId);
    List<TaskDataSource> taskDataSources;
    long total;
    // Find datasource by user
    taskDataSources = taskDataSourceDao.findByCreateUserAndDatasource(user.getUserName(), clusterName, databaseName, tableName, page, size);
    total = taskDataSourceDao.countByCreateUserAndDatasource(user.getUserName(), clusterName, databaseName, tableName);
    List<Application> applicationList = taskDataSources.stream().map(jobDataSource -> jobDataSource.getTask().getApplication()).collect(Collectors.toList());
    GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
    List<ApplicationResponse> applicationResponses = new ArrayList<>();
    for (Application application : applicationList) {
        List<Task> tasks = taskDao.findByApplication(application);
        ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
        if (application.getCreateUser().equals(user.getUserName()) || application.getExecuteUser().equals(user.getUserName())) {
            response.setKillOption(true);
        } else {
            response.setKillOption(false);
        }
        applicationResponses.add(response);
    }
    getAllResponse.setData(applicationResponses);
    getAllResponse.setTotal(total);
    List<String> applicationIdList = getAllResponse.getData().stream().map(ApplicationResponse::getApplicationId).collect(Collectors.toList());
    LOGGER.info("Succeed to find applications. size: {}, id of applications: {}", total, applicationIdList);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}

Example 17 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class JobServiceImpl method getTaskLog.

@Override
public GeneralResponse<?> getTaskLog(Long taskId, String clusterName) throws UnExpectedRequestException {
    Task task = taskDao.findById(taskId);
    if (task == null) {
        throw new UnExpectedRequestException("{&JOB_ID_DOES_NOT_EXIST}");
    }
    ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
    if (clusterInfo == null) {
        throw new UnExpectedRequestException("Cluster info {&DOES_NOT_EXIST}");
    }
    LogResult logResult;
    String proxyUser = task.getTaskProxyUser();
    try {
        logResult = monitorManager.getTaskPartialLog(task.getTaskRemoteId(), 0, StringUtils.isNotBlank(proxyUser) ? proxyUser : task.getApplication().getExecuteUser(), clusterInfo.getLinkisAddress(), clusterName);
    } catch (LogPartialException | ClusterInfoNotConfigException e) {
        throw new UnExpectedRequestException(e.getMessage());
    }
    LOGGER.info("Succeed to get task log, task_id: {}, cluster_id: {}", taskId, clusterName);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_TASK_LOG}", logResult.getLog());
}

Example 18 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class TaskServiceImpl method getTaskDetail.

@Override
public GeneralResponse<?> getTaskDetail(Long taskId) throws UnExpectedRequestException {
    Task taskInDb = taskDao.findById(taskId);
    if (taskInDb == null) {
        throw new UnExpectedRequestException("Job id [" + taskId + "] {&DOES_NOT_EXIST}");
    }
    // FIXME:
    List<TaskRuleAlarmConfig> distinct;
    Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(taskInDb.getTaskRuleSimples().size());
    for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
        distinct = taskRuleSimple.getTaskRuleAlarmConfigList().stream().distinct().collect(Collectors.toList());
        taskRuleSimple.setTaskRuleAlarmConfigList(distinct);
        taskRuleSimples.add(taskRuleSimple);
    }
    taskInDb.setTaskRuleSimples(taskRuleSimples);
    // Find single table verification rules
    List<TaskRuleSimple> singleRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.SINGLE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of single table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> singleRuleDataSourceMap = new HashMap<>(singleRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : singleRuleIds) {
        singleRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find custom table verification rules
    List<TaskRuleSimple> customRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of custom table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> customRuleDataSourceMap = new HashMap<>(customRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : customRuleIds) {
        customRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find multi-table verification rules
    List<TaskRuleSimple> multiRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of multi-table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> multiRuleDataSourceMap = new HashMap<>(multiRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : multiRuleIds) {
        multiRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find file table verification rules
    List<TaskRuleSimple> fileRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of file table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> fileRuleDataSourceMap = new HashMap<>(fileRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : fileRuleIds) {
        fileRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    List<Long> allRuleIds = new ArrayList<>();
    for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
        allRuleIds.add(taskRuleSimple.getRuleId());
        if (taskRuleSimple.getChildRuleSimple() != null) {
            allRuleIds.add(taskRuleSimple.getChildRuleSimple().getRuleId());
        }
    }
    List<TaskResult> allTaskResult = taskResultDao.findByApplicationIdAndRuleIn(taskInDb.getApplication().getId(), allRuleIds);
    Map<Long, List<TaskResult>> allResultMap = new HashMap<>(allTaskResult.size());
    for (TaskResult taskResult : allTaskResult) {
        if (allResultMap.get(taskResult.getRuleId()) != null) {
            allResultMap.get(taskResult.getRuleId()).add(taskResult);
        } else {
            List<TaskResult> taskResults = new ArrayList<>(1);
            taskResults.add(taskResult);
            allResultMap.put(taskResult.getRuleId(), taskResults);
        }
    }
    TaskCheckResultResponse taskCheckResultResponse = new TaskCheckResultResponse(taskInDb, singleRuleDataSourceMap, customRuleDataSourceMap, multiRuleDataSourceMap, fileRuleDataSourceMap, allResultMap);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_TASK_DETAIL}", taskCheckResultResponse);
}

Example 19 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method commonExecution.

@Override
public ApplicationTaskSimpleResponse commonExecution(List<Rule> rules, StringBuffer partition, String executionUser, String nodeName, String startupParam, String clusterName, String setFlag, Map<String, String> execParams, Application newApplication, Date date, StringBuffer runDate) throws RuleVariableNotSupportException, JobSubmitException, RuleVariableNotFoundException, ArgumentException, ConvertException, DataQualityTaskException, TaskTypeException, ClusterInfoNotConfigException, SystemConfigException, UnExpectedRequestException, MetaDataAcquireFailedException, IOException, TaskNotExistException, SemanticException, ParseException, DataSourceMoveException, DataSourceOverSizeException, org.apache.hadoop.hive.ql.parse.ParseException, BothNullDatasourceException, RightNullDatasourceException, LeftNullDatasourceException, java.text.ParseException {
    // current user
    String userName = executionUser;
    // Generate database name.
    String database = generateDatabase(userName);
    LOGGER.info("Succeed to generate database_name: {}", database);
    // Save application
    newApplication.setRuleSize(rules.size());
    newApplication.setSavedDb(database);
    Application saveApplication = applicationDao.saveApplication(newApplication);
    List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
    List<Rule> fileRules = new ArrayList<>();
    Map<Long, Map> dataSourceMysqlConnect = new HashMap<>(2);
    for (Iterator<Rule> iterator = rules.iterator(); iterator.hasNext(); ) {
        Rule currentRule = iterator.next();
        if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
            // Replace with execution parameter and parse datasource to save.
            customReSaveDateSource(currentRule, execParams, clusterName, date);
        }
        List<Map<String, String>> mappingCols = new ArrayList<>();
        getMappingCols(currentRule, mappingCols);
        // Check datasource before submit job.
        try {
            checkDatasource(currentRule, userName, partition, mappingCols, nodeName, clusterName, dataSourceMysqlConnect);
        } catch (BothNullDatasourceException e) {
            Task taskInDb = taskDao.save(new Task(newApplication, newApplication.getSubmitTime(), TaskStatusEnum.SUCCEED.getCode()));
            taskInDb.setClusterName(clusterName);
            TaskRuleSimple taskRuleSimple = new TaskRuleSimple(currentRule, taskInDb);
            Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
            Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
            taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
            for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
                taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
            }
            taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
            taskInDb.setTaskDataSources(taskDataSources);
            taskInDb.setTaskRuleSimples(taskRuleSimples);
            TaskResult taskResult = new TaskResult();
            taskResult.setApplicationId(newApplication.getId());
            taskResult.setCreateTime(newApplication.getSubmitTime());
            taskResult.setRuleId(currentRule.getId());
            taskResult.setResultType("Long");
            taskResult.setValue(0 + "");
            taskResultDao.saveTaskResult(taskResult);
            taskDao.save(taskInDb);
            iterator.remove();
            if (!iterator.hasNext()) {
                throw e;
            }
            continue;
        }
        if (currentRule.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())) {
            fileRules.add(currentRule);
            LOGGER.info("Succeed to find file rule. Rule: {}", currentRule.getId() + " " + currentRule.getName());
            iterator.remove();
        }
    }
    String submitTime = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(date);
    // General task.
    if (!rules.isEmpty()) {
        taskSubmitResults.addAll(executionManager.submitApplication(rules, nodeName, submitTime, userName, database, partition, date, saveApplication, clusterName, startupParam, setFlag, execParams, runDate, dataSourceMysqlConnect));
    }
    // Execute file rule task and save task result.
    if (!fileRules.isEmpty()) {
        taskSubmitResults.add(executionManager.executeFileRule(fileRules, submitTime, saveApplication, userName, clusterName, runDate));
    }
    saveApplication.setTotalTaskNum(taskSubmitResults.size());
    LOGGER.info("Succeed to submit application. result: {}", taskSubmitResults);
    Application applicationInDb = applicationDao.saveApplication(saveApplication);
    LOGGER.info("Succeed to save application. application: {}", applicationInDb);
    return new ApplicationTaskSimpleResponse(taskSubmitResults);
}

Example 20 with Task

use of com.webank.wedatasphere.qualitis.entity.Task in project Qualitis by WeBankFinTech.

the class TaskChecker method writeDb.

private void writeDb(JobChecker jobChecker, String newStatus, Integer errCode) {
    Task taskInDb = taskDao.findByRemoteTaskIdAndClusterName(jobChecker.getTaskId(), jobChecker.getClusterName());
    Application applicationInDb = applicationDao.findById(jobChecker.getApplicationId());
    if (newStatus.equals(TaskStatusEnum.FAILED.getState())) {
        /*
             * 1.Modify end time of job
             * 2.Modify task finish time and failed num if last job
             * */
        taskInDb.setEndTime(new DateTime(new Date()).toString(PRINT_TIME_FORMAT));
        taskInDb.setTaskComment(errCode == null ? ApplicationCommentEnum.UNKNOWN_ERROR_ISSUES.getCode() : ERR_CODE_TYPE.get(errCode));
        modifyJobStatus(taskInDb, newStatus);
        taskDao.save(taskInDb);
        applicationInDb.setApplicationComment(errCode == null ? ApplicationCommentEnum.UNKNOWN_ERROR_ISSUES.getCode() : ERR_CODE_TYPE.get(errCode));
        checkIfLastJob(applicationInDb, false, false, false);
    } else if (newStatus.equals(TaskStatusEnum.SUCCEED.getState())) {
        /*
             * 1.Modify end time of job
             * 2.Modify task finish time and succeed num if last job
             * */
        taskInDb.setEndTime(new DateTime(new Date()).toString(PRINT_TIME_FORMAT));
        boolean isPass;
        boolean finish;
        if (passCheckOut(jobChecker.getApplicationId(), taskInDb)) {
            modifyJobStatus(taskInDb, TaskStatusEnum.PASS_CHECKOUT.getState());
            isPass = true;
            finish = true;
        } else {
            if (taskInDb.getAbortOnFailure() != null && taskInDb.getAbortOnFailure()) {
                modifyJobStatus(taskInDb, TaskStatusEnum.FAILED.getState());
                taskInDb.setTaskComment(ApplicationCommentEnum.DIFF_DATA_ISSUES.getCode());
                applicationInDb.setApplicationComment(ApplicationCommentEnum.DIFF_DATA_ISSUES.getCode());
                finish = false;
            } else {
                modifyJobStatus(taskInDb, TaskStatusEnum.FAIL_CHECKOUT.getState());
                finish = true;
            }
            isPass = false;
        }
        taskDao.save(taskInDb);
        checkIfLastJob(applicationInDb, finish, isPass, false);
    } else if (newStatus.equals(TaskStatusEnum.CANCELLED.getState())) {
        modifyJobStatus(taskInDb, newStatus);
        taskDao.save(taskInDb);
        applicationInDb.setApplicationComment(ApplicationCommentEnum.TIMEOUT_KILL.getCode());
        checkIfLastJob(applicationInDb, false, false, false);
    } else {
        modifyJobStatus(taskInDb, newStatus);
        taskDao.save(taskInDb);
    }
}