Examples with TaskRuleAlarmConfig com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig used on opensource projects

Example 1 with TaskRuleAlarmConfig

use of com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method modifyTaskStatus.

private int modifyTaskStatus(List<TaskRuleAlarmConfig> alarmConfigs, Task taskInDb, List<TaskResult> taskResults, int successRule) {
    boolean rulePass = true;
    for (TaskRuleAlarmConfig alarmConfig : alarmConfigs) {
        TaskResult taskResult = taskResults.stream().filter(taskResultInDb -> taskResultInDb.getRuleMetricId().equals(alarmConfig.getRuleMetric().getId())).iterator().next();
        if (!FilePassUtil.pass(alarmConfig, taskResult, taskResultDao)) {
            rulePass = false;
            alarmConfig.setStatus(AlarmConfigStatusEnum.NOT_PASS.getCode());
            if (alarmConfig.getDeleteFailCheckResult() != null && alarmConfig.getDeleteFailCheckResult()) {
                taskResult.setSaveResult(false);
                taskResultDao.saveTaskResult(taskResult);
            }
        } else {
            alarmConfig.setStatus(AlarmConfigStatusEnum.PASS.getCode());
        }
    }
    if (rulePass) {
        successRule++;
    } else if (taskInDb.getAbortOnFailure() != null && taskInDb.getAbortOnFailure()) {
        taskInDb.setStatus(TaskStatusEnum.FAILED.getCode());
    } else {
        taskInDb.setStatus(TaskStatusEnum.FAIL_CHECKOUT.getCode());
        taskInDb.setProgress(Double.parseDouble("1"));
    }
    return successRule;
}

Example 2 with TaskRuleAlarmConfig

use of com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method getApplicationResult.

@Override
public GeneralResponse<?> getApplicationResult(String applicationId) throws UnExpectedRequestException {
    // Find application by applicationId
    Application applicationInDb = applicationDao.findById(applicationId);
    if (applicationInDb == null) {
        throw new UnExpectedRequestException(String.format("ApplicationId: %s {&DOES_NOT_EXIST}", applicationId));
    }
    Integer passNum = 0;
    Integer failedNum = 0;
    Integer notPassNum = 0;
    StringBuilder passMessage = new StringBuilder();
    StringBuilder failedMessage = new StringBuilder();
    StringBuilder notPassMessage = new StringBuilder();
    passMessage.append("The rules following below are pass(以下规则已通过校验): [");
    failedMessage.append("The rules following below are failed(以下规则已失败): [");
    notPassMessage.append("The rules following below are failed(以下规则未通过校验): [");
    // Find all tasks by application
    List<Task> tasks = taskDao.findByApplication(applicationInDb);
    // Find task rule simple in all tasks
    for (Task task : tasks) {
        // Add failed num, pass num if task status equals to Succeed, Failed
        if (task.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
            failedNum += task.getTaskRuleSimples().size();
            generateFailedMessage(task, failedMessage);
        } else if (task.getStatus().equals(TaskStatusEnum.PASS_CHECKOUT.getCode())) {
            passNum += task.getTaskRuleSimples().size();
            generatePassMessage(task, passMessage);
        } else if (task.getStatus().equals(TaskStatusEnum.FAIL_CHECKOUT.getCode())) {
            // Find not pass task if task status equals to failed_checkout
            for (TaskRuleSimple taskRuleSimple : task.getTaskRuleSimples()) {
                Boolean isPass = true;
                for (TaskRuleAlarmConfig taskRuleAlarmConfig : taskRuleSimple.getTaskRuleAlarmConfigList()) {
                    if (taskRuleAlarmConfig.getStatus().equals(AlarmConfigStatusEnum.NOT_PASS.getCode())) {
                        isPass = false;
                        break;
                    }
                }
                if (isPass) {
                    passNum++;
                    passMessage.append(taskRuleSimple.getRuleName()).append(", ");
                } else {
                    notPassNum++;
                    notPassMessage.append(taskRuleSimple.getRuleName()).append(", ");
                }
            }
        } else {
            throw new UnExpectedRequestException(String.format("ApplicationId: %s {&DOES_NOT_FINISHED_YET}", applicationId));
        }
    }
    passMessage.append("]").append(System.lineSeparator());
    failedMessage.append("]").append(System.lineSeparator());
    notPassMessage.append("]").append(System.lineSeparator());
    String resultMessage = passMessage.toString() + failedMessage.toString() + notPassMessage.toString();
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATION_RESULT}", new ApplicationResultResponse(passNum, failedNum, notPassNum, resultMessage));
}

Example 3 with TaskRuleAlarmConfig

use of com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method joinAlarmConfig.

private void joinAlarmConfig(List<ExcelResult> results, TaskRuleSimple taskRuleSimple, ExcelResult excelResult, UploadResultRequest request, StringBuffer checkTemplateStr, StringBuffer resultStr) {
    for (TaskRuleAlarmConfig taskRuleAlarmConfig : taskRuleSimple.getTaskRuleAlarmConfigList()) {
        Integer checkTemplate = taskRuleAlarmConfig.getCheckTemplate();
        String checkTemplateName = CheckTemplateEnum.getCheckTemplateName(checkTemplate);
        String outputName = taskRuleAlarmConfig.getOutputName();
        String compare = CompareTypeEnum.getCompareTypeName(taskRuleAlarmConfig.getCompareType());
        Double threshold = taskRuleAlarmConfig.getThreshold();
        Integer statusCode = taskRuleAlarmConfig.getStatus();
        String status = AlarmConfigStatusEnum.getMessage(statusCode);
        RuleMetric ruleMetric = taskRuleAlarmConfig.getRuleMetric();
        TaskResult taskResult = taskResultDao.find(taskRuleSimple.getApplicationId(), taskRuleSimple.getRuleId(), ruleMetric != null ? ruleMetric.getId() : -1);
        String resultValue = "";
        Long runDate = null;
        if (taskResult != null) {
            resultValue = StringUtils.isNotBlank(taskResult.getValue()) ? taskResult.getValue() : "0";
            runDate = taskResult.getRunDate();
        }
        if (request.getStatus().contains(statusCode)) {
            checkTemplateStr.append(checkTemplateName).append(StringUtils.isNotBlank(compare) ? compare : "").append(" ").append(threshold).append(" ").append(status);
            ExcelResult excelAlarmConfig = new ExcelResult();
            BeanUtils.copyProperties(excelResult, excelAlarmConfig);
            excelAlarmConfig.setRuleCheckTemplates(checkTemplateStr.toString());
            resultStr.append(outputName).append(": ").append(resultValue);
            excelAlarmConfig.setHistoryResult(resultStr.toString());
            excelAlarmConfig.setCreateTime(runDate != null ? runDate.toString() : "");
            excelAlarmConfig.setExecutionUser(taskRuleSimple.getExecuteUser());
            results.add(excelAlarmConfig);
            checkTemplateStr.delete(0, checkTemplateStr.length());
            resultStr.delete(0, checkTemplateStr.length());
        }
    }
}

Example 4 with TaskRuleAlarmConfig

use of com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig in project Qualitis by WeBankFinTech.

the class TaskServiceImpl method getTaskDetail.

@Override
public GeneralResponse<?> getTaskDetail(Long taskId) throws UnExpectedRequestException {
    Task taskInDb = taskDao.findById(taskId);
    if (taskInDb == null) {
        throw new UnExpectedRequestException("Job id [" + taskId + "] {&DOES_NOT_EXIST}");
    }
    // FIXME:
    List<TaskRuleAlarmConfig> distinct;
    Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(taskInDb.getTaskRuleSimples().size());
    for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
        distinct = taskRuleSimple.getTaskRuleAlarmConfigList().stream().distinct().collect(Collectors.toList());
        taskRuleSimple.setTaskRuleAlarmConfigList(distinct);
        taskRuleSimples.add(taskRuleSimple);
    }
    taskInDb.setTaskRuleSimples(taskRuleSimples);
    // Find single table verification rules
    List<TaskRuleSimple> singleRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.SINGLE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of single table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> singleRuleDataSourceMap = new HashMap<>(singleRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : singleRuleIds) {
        singleRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find custom table verification rules
    List<TaskRuleSimple> customRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of custom table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> customRuleDataSourceMap = new HashMap<>(customRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : customRuleIds) {
        customRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find multi-table verification rules
    List<TaskRuleSimple> multiRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of multi-table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> multiRuleDataSourceMap = new HashMap<>(multiRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : multiRuleIds) {
        multiRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find file table verification rules
    List<TaskRuleSimple> fileRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of file table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> fileRuleDataSourceMap = new HashMap<>(fileRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : fileRuleIds) {
        fileRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    List<Long> allRuleIds = new ArrayList<>();
    for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
        allRuleIds.add(taskRuleSimple.getRuleId());
        if (taskRuleSimple.getChildRuleSimple() != null) {
            allRuleIds.add(taskRuleSimple.getChildRuleSimple().getRuleId());
        }
    }
    List<TaskResult> allTaskResult = taskResultDao.findByApplicationIdAndRuleIn(taskInDb.getApplication().getId(), allRuleIds);
    Map<Long, List<TaskResult>> allResultMap = new HashMap<>(allTaskResult.size());
    for (TaskResult taskResult : allTaskResult) {
        if (allResultMap.get(taskResult.getRuleId()) != null) {
            allResultMap.get(taskResult.getRuleId()).add(taskResult);
        } else {
            List<TaskResult> taskResults = new ArrayList<>(1);
            taskResults.add(taskResult);
            allResultMap.put(taskResult.getRuleId(), taskResults);
        }
    }
    TaskCheckResultResponse taskCheckResultResponse = new TaskCheckResultResponse(taskInDb, singleRuleDataSourceMap, customRuleDataSourceMap, multiRuleDataSourceMap, fileRuleDataSourceMap, allResultMap);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_TASK_DETAIL}", taskCheckResultResponse);
}

Example 5 with TaskRuleAlarmConfig

use of com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig in project Qualitis by WeBankFinTech.

the class TaskChecker method checkTaskRuleSimplePass.

@Transactional(rollbackFor = { RuntimeException.class, UnExpectedRequestException.class })
private Boolean checkTaskRuleSimplePass(String applicationId, TaskRuleSimple taskRuleSimple) {
    Boolean passFlag = true;
    List<TaskResult> taskResults = taskResultDao.findByApplicationAndRule(applicationId, taskRuleSimple.getRuleId());
    for (TaskResult taskResult : taskResults) {
        List<TaskRuleAlarmConfig> taskRuleAlarmConfigList = taskRuleSimple.getTaskRuleAlarmConfigList();
        Long ruleMetricId = taskResult.getRuleMetricId();
        if (ruleMetricId != null && ruleMetricId.longValue() != -1) {
            taskRuleAlarmConfigList = taskRuleAlarmConfigList.stream().filter(taskRuleAlarmConfig -> taskRuleAlarmConfig.getRuleMetric().getId().equals(ruleMetricId)).collect(Collectors.toList());
        }
        for (TaskRuleAlarmConfig taskRuleAlarmConfig : taskRuleAlarmConfigList) {
            if (PassUtil.notSafe(applicationId, taskRuleSimple.getRuleId(), taskRuleAlarmConfig, taskResult, taskResultDao)) {
                taskRuleAlarmConfig.setStatus(AlarmConfigStatusEnum.PASS.getCode());
            } else {
                passFlag = false;
                taskRuleAlarmConfig.setStatus(AlarmConfigStatusEnum.NOT_PASS.getCode());
                if (taskRuleSimple.getRuleType().equals(RuleTemplateTypeEnum.CUSTOM.getCode())) {
                    if (taskRuleAlarmConfig.getDeleteFailCheckResult() != null && true == taskRuleAlarmConfig.getDeleteFailCheckResult().booleanValue()) {
                        taskResult.setSaveResult(false);
                        taskResultDao.saveTaskResult(taskResult);
                    }
                } else {
                    if (taskRuleSimple.getDeleteFailCheckResult() != null && true == taskRuleSimple.getDeleteFailCheckResult().booleanValue()) {
                        taskResult.setSaveResult(false);
                        taskResultDao.saveTaskResult(taskResult);
                    }
                }
            }
        }
    }
    return passFlag;
}