Examples with TaskResult com.webank.wedatasphere.qualitis.entity.TaskResult used on opensource projects

Example 1 with TaskResult

use of com.webank.wedatasphere.qualitis.entity.TaskResult in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method modifyTaskStatus.

private int modifyTaskStatus(List<TaskRuleAlarmConfig> alarmConfigs, Task taskInDb, List<TaskResult> taskResults, int successRule) {
    boolean rulePass = true;
    for (TaskRuleAlarmConfig alarmConfig : alarmConfigs) {
        TaskResult taskResult = taskResults.stream().filter(taskResultInDb -> taskResultInDb.getRuleMetricId().equals(alarmConfig.getRuleMetric().getId())).iterator().next();
        if (!FilePassUtil.pass(alarmConfig, taskResult, taskResultDao)) {
            rulePass = false;
            alarmConfig.setStatus(AlarmConfigStatusEnum.NOT_PASS.getCode());
            if (alarmConfig.getDeleteFailCheckResult() != null && alarmConfig.getDeleteFailCheckResult()) {
                taskResult.setSaveResult(false);
                taskResultDao.saveTaskResult(taskResult);
            }
        } else {
            alarmConfig.setStatus(AlarmConfigStatusEnum.PASS.getCode());
        }
    }
    if (rulePass) {
        successRule++;
    } else if (taskInDb.getAbortOnFailure() != null && taskInDb.getAbortOnFailure()) {
        taskInDb.setStatus(TaskStatusEnum.FAILED.getCode());
    } else {
        taskInDb.setStatus(TaskStatusEnum.FAIL_CHECKOUT.getCode());
        taskInDb.setProgress(Double.parseDouble("1"));
    }
    return successRule;
}

Example 2 with TaskResult

use of com.webank.wedatasphere.qualitis.entity.TaskResult in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method executeFileRule.

@Override
public TaskSubmitResult executeFileRule(List<Rule> fileRules, String submitTime, Application application, String user, String clusterName, StringBuffer runDate) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    LOGGER.info("Start to execute file rule task and save check result.");
    Task taskInDb = taskDao.save(new Task(application, submitTime, TaskStatusEnum.SUBMITTED.getCode()));
    Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
    Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
    int totalRules = fileRules.size();
    int successRule = 0;
    for (Rule rule : fileRules) {
        if (rule.getAbortOnFailure() != null) {
            taskInDb.setAbortOnFailure(rule.getAbortOnFailure());
        }
        TaskRuleSimple taskRuleSimple = new TaskRuleSimple(rule, taskInDb, httpServletRequest.getHeader("Content-Language"));
        taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
        RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
        taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
        // Check rule datasource: 1) table 2) partition.
        if (StringUtils.isEmpty(ruleDataSource.getFilter())) {
            TableStatisticsInfo result;
            try {
                String proxyUser = ruleDataSource.getProxyUser();
                result = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
            } catch (RestClientException e) {
                LOGGER.error("Failed to get table statistics with linkis api.", e);
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            if (result == null) {
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            String fullSize = result.getTableSize();
            List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getTableFileCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
            successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
        } else {
            PartitionStatisticsInfo result;
            try {
                String proxyUser = ruleDataSource.getProxyUser();
                result = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(DateExprReplaceUtil.replaceFilter(new Date(), ruleDataSource.getFilter())), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
            } catch (RestClientException e) {
                LOGGER.error("Failed to get table statistics with linkis api.", e);
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            if (result == null) {
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            String fullSize = result.getPartitionSize();
            List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getPartitionChildCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
            successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
        }
        if (taskInDb.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
            break;
        }
    }
    taskInDb.setTaskDataSources(taskDataSources);
    taskInDb.setTaskRuleSimples(taskRuleSimples);
    // Update task status
    taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
    if (totalRules == successRule) {
        taskInDb.setStatus(TaskStatusEnum.PASS_CHECKOUT.getCode());
        taskInDb.setProgress(Double.parseDouble("1"));
    }
    taskDao.save(taskInDb);
    LOGGER.info("Finished to execute file rule task and save check result.");
    TaskSubmitResult taskSubmitResult = new TaskSubmitResult();
    taskSubmitResult.setApplicationId(application.getId());
    taskSubmitResult.setClusterName(clusterName);
    return taskSubmitResult;
}

Example 3 with TaskResult

use of com.webank.wedatasphere.qualitis.entity.TaskResult in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method saveTaskRusult.

private List<TaskResult> saveTaskRusult(String fullSize, double fileCount, Application application, String submitTime, Rule rule, Set<AlarmConfig> alarmConfig, String runDate) throws UnExpectedRequestException {
    double number = Double.parseDouble(fullSize.split(" ")[0]);
    String unit = fullSize.split(" ")[1];
    // Save task result.
    List<TaskResult> taskResults = new ArrayList<>();
    List<Integer> fileOutputNames = alarmConfig.stream().map(AlarmConfig::getFileOutputName).distinct().collect(Collectors.toList());
    for (Integer fileOutputName : fileOutputNames) {
        AlarmConfig currentAlarmConfig = alarmConfig.stream().filter(alarmConfigSetting -> alarmConfigSetting.getFileOutputName().equals(fileOutputName)).iterator().next();
        RuleMetric ruleMetric = currentAlarmConfig.getRuleMetric();
        if (ruleMetric == null) {
            throw new UnExpectedRequestException("File rule metric {&CAN_NOT_BE_NULL_OR_EMPTY}");
        }
        TaskResult taskResult;
        if (StringUtils.isNotBlank(runDate)) {
            TaskResult existTaskResult = taskResultDao.find(runDate, rule.getId(), ruleMetric.getId());
            if (existTaskResult != null) {
                taskResult = existTaskResult;
            } else {
                taskResult = new TaskResult();
                Date runRealDate = null;
                try {
                    runRealDate = new SimpleDateFormat("yyyyMMdd").parse(runDate);
                } catch (ParseException e) {
                    String errorMsg = "Parse date string with run date failed. Exception message: " + e.getMessage();
                    LOGGER.error(errorMsg);
                    throw new UnExpectedRequestException(errorMsg);
                }
                taskResult.setRunDate(runRealDate.getTime());
            }
        } else {
            taskResult = new TaskResult();
        }
        taskResult.setApplicationId(application.getId());
        taskResult.setCreateTime(submitTime);
        taskResult.setRuleId(rule.getId());
        taskResult.setRuleMetricId(ruleMetric.getId());
        if (fileOutputName.equals(FileOutputNameEnum.FILE_COUNT.getCode())) {
            taskResult.setResultType("int");
            taskResult.setValue(fileCount + "");
        } else if (fileOutputName.equals(FileOutputNameEnum.DIR_SIZE.getCode())) {
            String alarmConfigUnit = FileOutputUnitEnum.fileOutputUnit(currentAlarmConfig.getFileOutputUnit());
            taskResult.setResultType(alarmConfigUnit);
            taskResult.setValue(UnitTransfer.alarmconfigToTaskResult(number, alarmConfigUnit, unit.toUpperCase()) + "");
        } else {
            throw new UnExpectedRequestException("Unknown file output name.");
        }
        taskResults.add(taskResultDao.saveTaskResult(taskResult));
    }
    return taskResults;
}

Example 4 with TaskResult

use of com.webank.wedatasphere.qualitis.entity.TaskResult in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method joinAlarmConfig.

private void joinAlarmConfig(List<ExcelResult> results, TaskRuleSimple taskRuleSimple, ExcelResult excelResult, UploadResultRequest request, StringBuffer checkTemplateStr, StringBuffer resultStr) {
    for (TaskRuleAlarmConfig taskRuleAlarmConfig : taskRuleSimple.getTaskRuleAlarmConfigList()) {
        Integer checkTemplate = taskRuleAlarmConfig.getCheckTemplate();
        String checkTemplateName = CheckTemplateEnum.getCheckTemplateName(checkTemplate);
        String outputName = taskRuleAlarmConfig.getOutputName();
        String compare = CompareTypeEnum.getCompareTypeName(taskRuleAlarmConfig.getCompareType());
        Double threshold = taskRuleAlarmConfig.getThreshold();
        Integer statusCode = taskRuleAlarmConfig.getStatus();
        String status = AlarmConfigStatusEnum.getMessage(statusCode);
        RuleMetric ruleMetric = taskRuleAlarmConfig.getRuleMetric();
        TaskResult taskResult = taskResultDao.find(taskRuleSimple.getApplicationId(), taskRuleSimple.getRuleId(), ruleMetric != null ? ruleMetric.getId() : -1);
        String resultValue = "";
        Long runDate = null;
        if (taskResult != null) {
            resultValue = StringUtils.isNotBlank(taskResult.getValue()) ? taskResult.getValue() : "0";
            runDate = taskResult.getRunDate();
        }
        if (request.getStatus().contains(statusCode)) {
            checkTemplateStr.append(checkTemplateName).append(StringUtils.isNotBlank(compare) ? compare : "").append(" ").append(threshold).append(" ").append(status);
            ExcelResult excelAlarmConfig = new ExcelResult();
            BeanUtils.copyProperties(excelResult, excelAlarmConfig);
            excelAlarmConfig.setRuleCheckTemplates(checkTemplateStr.toString());
            resultStr.append(outputName).append(": ").append(resultValue);
            excelAlarmConfig.setHistoryResult(resultStr.toString());
            excelAlarmConfig.setCreateTime(runDate != null ? runDate.toString() : "");
            excelAlarmConfig.setExecutionUser(taskRuleSimple.getExecuteUser());
            results.add(excelAlarmConfig);
            checkTemplateStr.delete(0, checkTemplateStr.length());
            resultStr.delete(0, checkTemplateStr.length());
        }
    }
}

Example 5 with TaskResult

use of com.webank.wedatasphere.qualitis.entity.TaskResult in project Qualitis by WeBankFinTech.

the class TaskServiceImpl method getTaskDetail.

@Override
public GeneralResponse<?> getTaskDetail(Long taskId) throws UnExpectedRequestException {
    Task taskInDb = taskDao.findById(taskId);
    if (taskInDb == null) {
        throw new UnExpectedRequestException("Job id [" + taskId + "] {&DOES_NOT_EXIST}");
    }
    // FIXME:
    List<TaskRuleAlarmConfig> distinct;
    Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(taskInDb.getTaskRuleSimples().size());
    for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
        distinct = taskRuleSimple.getTaskRuleAlarmConfigList().stream().distinct().collect(Collectors.toList());
        taskRuleSimple.setTaskRuleAlarmConfigList(distinct);
        taskRuleSimples.add(taskRuleSimple);
    }
    taskInDb.setTaskRuleSimples(taskRuleSimples);
    // Find single table verification rules
    List<TaskRuleSimple> singleRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.SINGLE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of single table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> singleRuleDataSourceMap = new HashMap<>(singleRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : singleRuleIds) {
        singleRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find custom table verification rules
    List<TaskRuleSimple> customRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of custom table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> customRuleDataSourceMap = new HashMap<>(customRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : customRuleIds) {
        customRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find multi-table verification rules
    List<TaskRuleSimple> multiRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of multi-table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> multiRuleDataSourceMap = new HashMap<>(multiRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : multiRuleIds) {
        multiRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find file table verification rules
    List<TaskRuleSimple> fileRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of file table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> fileRuleDataSourceMap = new HashMap<>(fileRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : fileRuleIds) {
        fileRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    List<Long> allRuleIds = new ArrayList<>();
    for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
        allRuleIds.add(taskRuleSimple.getRuleId());
        if (taskRuleSimple.getChildRuleSimple() != null) {
            allRuleIds.add(taskRuleSimple.getChildRuleSimple().getRuleId());
        }
    }
    List<TaskResult> allTaskResult = taskResultDao.findByApplicationIdAndRuleIn(taskInDb.getApplication().getId(), allRuleIds);
    Map<Long, List<TaskResult>> allResultMap = new HashMap<>(allTaskResult.size());
    for (TaskResult taskResult : allTaskResult) {
        if (allResultMap.get(taskResult.getRuleId()) != null) {
            allResultMap.get(taskResult.getRuleId()).add(taskResult);
        } else {
            List<TaskResult> taskResults = new ArrayList<>(1);
            taskResults.add(taskResult);
            allResultMap.put(taskResult.getRuleId(), taskResults);
        }
    }
    TaskCheckResultResponse taskCheckResultResponse = new TaskCheckResultResponse(taskInDb, singleRuleDataSourceMap, customRuleDataSourceMap, multiRuleDataSourceMap, fileRuleDataSourceMap, allResultMap);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_TASK_DETAIL}", taskCheckResultResponse);
}