Examples with TaskRuleSimple com.webank.wedatasphere.qualitis.entity.TaskRuleSimple used on opensource projects

Example 1 with TaskRuleSimple

use of com.webank.wedatasphere.qualitis.entity.TaskRuleSimple in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method saveJobRuleSimpleAndJobDataSource.

/**
 * Save task rule simple and rule datasource
 * @param ruleList
 * @param task
 */
private void saveJobRuleSimpleAndJobDataSource(List<TaskRule> ruleList, Task task) {
    for (TaskRule rule : ruleList) {
        if (rule.getChildRuleId() != null) {
            // Save parent task rule simple
            TaskRuleSimple parentRuleSimple = taskRuleSimpleRepository.save(new TaskRuleSimple(rule, task, true, null));
            // Save child task rule simple
            TaskRuleSimple childRuleSimple = taskRuleSimpleRepository.save(new TaskRuleSimple(rule, task, false, parentRuleSimple));
            task.getTaskRuleSimples().add(parentRuleSimple);
        } else {
            TaskRuleSimple taskRuleSimple = taskRuleSimpleRepository.save(new TaskRuleSimple(rule, task));
            LOGGER.info("Succeed to save task_rule: task_rule_id: {}, rule_name: {}", taskRuleSimple.getId(), taskRuleSimple.getRuleName());
            task.getTaskRuleSimples().add(taskRuleSimple);
        }
        for (TaskRuleDataSource ruleDataSource : rule.getTaskRuleDataSourceList()) {
            TaskDataSource taskDataSource = taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, task));
            LOGGER.info("Succeed to save task_datasource: task_datasource_id: {}, cluster: {}, database: {}, table: {}", taskDataSource.getId(), taskDataSource.getClusterName(), taskDataSource.getDatabaseName(), taskDataSource.getTableName());
            task.getTaskDataSources().add(taskDataSource);
        }
    }
}

Example 2 with TaskRuleSimple

use of com.webank.wedatasphere.qualitis.entity.TaskRuleSimple in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method executeFileRule.

@Override
public TaskSubmitResult executeFileRule(List<Rule> fileRules, String submitTime, Application application, String user, String clusterName, StringBuffer runDate) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    LOGGER.info("Start to execute file rule task and save check result.");
    Task taskInDb = taskDao.save(new Task(application, submitTime, TaskStatusEnum.SUBMITTED.getCode()));
    Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
    Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
    int totalRules = fileRules.size();
    int successRule = 0;
    for (Rule rule : fileRules) {
        if (rule.getAbortOnFailure() != null) {
            taskInDb.setAbortOnFailure(rule.getAbortOnFailure());
        }
        TaskRuleSimple taskRuleSimple = new TaskRuleSimple(rule, taskInDb, httpServletRequest.getHeader("Content-Language"));
        taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
        RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
        taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
        // Check rule datasource: 1) table 2) partition.
        if (StringUtils.isEmpty(ruleDataSource.getFilter())) {
            TableStatisticsInfo result;
            try {
                String proxyUser = ruleDataSource.getProxyUser();
                result = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
            } catch (RestClientException e) {
                LOGGER.error("Failed to get table statistics with linkis api.", e);
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            if (result == null) {
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            String fullSize = result.getTableSize();
            List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getTableFileCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
            successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
        } else {
            PartitionStatisticsInfo result;
            try {
                String proxyUser = ruleDataSource.getProxyUser();
                result = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(DateExprReplaceUtil.replaceFilter(new Date(), ruleDataSource.getFilter())), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
            } catch (RestClientException e) {
                LOGGER.error("Failed to get table statistics with linkis api.", e);
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            if (result == null) {
                throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
            }
            String fullSize = result.getPartitionSize();
            List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getPartitionChildCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
            successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
        }
        if (taskInDb.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
            break;
        }
    }
    taskInDb.setTaskDataSources(taskDataSources);
    taskInDb.setTaskRuleSimples(taskRuleSimples);
    // Update task status
    taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
    if (totalRules == successRule) {
        taskInDb.setStatus(TaskStatusEnum.PASS_CHECKOUT.getCode());
        taskInDb.setProgress(Double.parseDouble("1"));
    }
    taskDao.save(taskInDb);
    LOGGER.info("Finished to execute file rule task and save check result.");
    TaskSubmitResult taskSubmitResult = new TaskSubmitResult();
    taskSubmitResult.setApplicationId(application.getId());
    taskSubmitResult.setClusterName(clusterName);
    return taskSubmitResult;
}

Example 3 with TaskRuleSimple

use of com.webank.wedatasphere.qualitis.entity.TaskRuleSimple in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method getApplicationResult.

@Override
public GeneralResponse<?> getApplicationResult(String applicationId) throws UnExpectedRequestException {
    // Find application by applicationId
    Application applicationInDb = applicationDao.findById(applicationId);
    if (applicationInDb == null) {
        throw new UnExpectedRequestException(String.format("ApplicationId: %s {&DOES_NOT_EXIST}", applicationId));
    }
    Integer passNum = 0;
    Integer failedNum = 0;
    Integer notPassNum = 0;
    StringBuilder passMessage = new StringBuilder();
    StringBuilder failedMessage = new StringBuilder();
    StringBuilder notPassMessage = new StringBuilder();
    passMessage.append("The rules following below are pass(以下规则已通过校验): [");
    failedMessage.append("The rules following below are failed(以下规则已失败): [");
    notPassMessage.append("The rules following below are failed(以下规则未通过校验): [");
    // Find all tasks by application
    List<Task> tasks = taskDao.findByApplication(applicationInDb);
    // Find task rule simple in all tasks
    for (Task task : tasks) {
        // Add failed num, pass num if task status equals to Succeed, Failed
        if (task.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
            failedNum += task.getTaskRuleSimples().size();
            generateFailedMessage(task, failedMessage);
        } else if (task.getStatus().equals(TaskStatusEnum.PASS_CHECKOUT.getCode())) {
            passNum += task.getTaskRuleSimples().size();
            generatePassMessage(task, passMessage);
        } else if (task.getStatus().equals(TaskStatusEnum.FAIL_CHECKOUT.getCode())) {
            // Find not pass task if task status equals to failed_checkout
            for (TaskRuleSimple taskRuleSimple : task.getTaskRuleSimples()) {
                Boolean isPass = true;
                for (TaskRuleAlarmConfig taskRuleAlarmConfig : taskRuleSimple.getTaskRuleAlarmConfigList()) {
                    if (taskRuleAlarmConfig.getStatus().equals(AlarmConfigStatusEnum.NOT_PASS.getCode())) {
                        isPass = false;
                        break;
                    }
                }
                if (isPass) {
                    passNum++;
                    passMessage.append(taskRuleSimple.getRuleName()).append(", ");
                } else {
                    notPassNum++;
                    notPassMessage.append(taskRuleSimple.getRuleName()).append(", ");
                }
            }
        } else {
            throw new UnExpectedRequestException(String.format("ApplicationId: %s {&DOES_NOT_FINISHED_YET}", applicationId));
        }
    }
    passMessage.append("]").append(System.lineSeparator());
    failedMessage.append("]").append(System.lineSeparator());
    notPassMessage.append("]").append(System.lineSeparator());
    String resultMessage = passMessage.toString() + failedMessage.toString() + notPassMessage.toString();
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATION_RESULT}", new ApplicationResultResponse(passNum, failedNum, notPassNum, resultMessage));
}

Example 4 with TaskRuleSimple

use of com.webank.wedatasphere.qualitis.entity.TaskRuleSimple in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method writeExcelFile.

private void writeExcelFile(File tmpFile, List<String> tables, UploadResultRequest request) throws UnExpectedRequestException {
    int sheetNo = 1;
    OutputStream tmpOutputStream = null;
    try {
        Path parentPath = Paths.get(tmpFile.getParent());
        if (Files.notExists(parentPath)) {
            tmpFile.getParentFile().mkdirs();
        }
        if (!tmpFile.exists()) {
            LOGGER.info("Start to create local file.");
            tmpFile.createNewFile();
        }
        tmpOutputStream = new FileOutputStream(tmpFile);
        ExcelWriter writer = new ExcelWriter(tmpOutputStream, ExcelTypeEnum.XLSX, true);
        // Find task with the start time and end time.
        for (String tableName : tables) {
            List<Task> tasks = taskDao.findWithSubmitTimeAndDatasource(request.getStartTime(), request.getEndTime(), request.getClusterName(), request.getDatabaseName(), tableName);
            List<TaskRuleSimple> taskRuleSimples = tasks.stream().map(Task::getTaskRuleSimples).flatMap(taskRuleSimpleSet -> taskRuleSimpleSet.stream()).distinct().collect(Collectors.toList());
            // Generate analysis result excel
            List<ExcelResult> results = new ArrayList<>(taskRuleSimples.size());
            for (TaskRuleSimple taskRuleSimple : taskRuleSimples) {
                ExcelResult excelResult = new ExcelResult();
                excelResult.setRuleName(taskRuleSimple.getRuleName());
                excelResult.setClusterName(request.getClusterName());
                excelResult.setDatabaseName(request.getDatabaseName());
                excelResult.setTableName(tableName);
                excelResult.setBeginTime(taskRuleSimple.getTask().getBeginTime());
                excelResult.setEndTime(taskRuleSimple.getTask().getEndTime());
                StringBuffer checkTemplateStr = new StringBuffer();
                StringBuffer resultStr = new StringBuffer();
                joinAlarmConfig(results, taskRuleSimple, excelResult, request, checkTemplateStr, resultStr);
                results.add(excelResult);
            }
            Sheet templateSheet = new Sheet(sheetNo++, 0, ExcelResult.class);
            templateSheet.setSheetName(tableName + "-" + ExcelSheetName.ANALYSIS_NAME);
            writer.write(results, templateSheet);
        }
        writer.finish();
    } catch (FileNotFoundException e) {
        LOGGER.error(e.getMessage(), e);
        throw new UnExpectedRequestException("{&FAILED_TO_CREATE_LOCAL_FILE}");
    } catch (IOException e) {
        LOGGER.error(e.getMessage(), e);
        throw new UnExpectedRequestException("{&FAILED_TO_CREATE_LOCAL_FILE}");
    } finally {
        try {
            tmpOutputStream.close();
        } catch (IOException e) {
            LOGGER.error(e.getMessage(), e);
        }
    }
}

Example 5 with TaskRuleSimple

use of com.webank.wedatasphere.qualitis.entity.TaskRuleSimple in project Qualitis by WeBankFinTech.

the class TaskServiceImpl method getTaskDetail.

@Override
public GeneralResponse<?> getTaskDetail(Long taskId) throws UnExpectedRequestException {
    Task taskInDb = taskDao.findById(taskId);
    if (taskInDb == null) {
        throw new UnExpectedRequestException("Job id [" + taskId + "] {&DOES_NOT_EXIST}");
    }
    // FIXME:
    List<TaskRuleAlarmConfig> distinct;
    Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(taskInDb.getTaskRuleSimples().size());
    for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
        distinct = taskRuleSimple.getTaskRuleAlarmConfigList().stream().distinct().collect(Collectors.toList());
        taskRuleSimple.setTaskRuleAlarmConfigList(distinct);
        taskRuleSimples.add(taskRuleSimple);
    }
    taskInDb.setTaskRuleSimples(taskRuleSimples);
    // Find single table verification rules
    List<TaskRuleSimple> singleRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.SINGLE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of single table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> singleRuleDataSourceMap = new HashMap<>(singleRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : singleRuleIds) {
        singleRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find custom table verification rules
    List<TaskRuleSimple> customRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of custom table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> customRuleDataSourceMap = new HashMap<>(customRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : customRuleIds) {
        customRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find multi-table verification rules
    List<TaskRuleSimple> multiRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of multi-table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> multiRuleDataSourceMap = new HashMap<>(multiRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : multiRuleIds) {
        multiRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    // Find file table verification rules
    List<TaskRuleSimple> fileRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
    // Find all datasources of file table verification rules, and save it in the map
    Map<TaskRuleSimple, List<TaskDataSource>> fileRuleDataSourceMap = new HashMap<>(fileRuleIds.size());
    for (TaskRuleSimple taskRuleSimple : fileRuleIds) {
        fileRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
    }
    List<Long> allRuleIds = new ArrayList<>();
    for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
        allRuleIds.add(taskRuleSimple.getRuleId());
        if (taskRuleSimple.getChildRuleSimple() != null) {
            allRuleIds.add(taskRuleSimple.getChildRuleSimple().getRuleId());
        }
    }
    List<TaskResult> allTaskResult = taskResultDao.findByApplicationIdAndRuleIn(taskInDb.getApplication().getId(), allRuleIds);
    Map<Long, List<TaskResult>> allResultMap = new HashMap<>(allTaskResult.size());
    for (TaskResult taskResult : allTaskResult) {
        if (allResultMap.get(taskResult.getRuleId()) != null) {
            allResultMap.get(taskResult.getRuleId()).add(taskResult);
        } else {
            List<TaskResult> taskResults = new ArrayList<>(1);
            taskResults.add(taskResult);
            allResultMap.put(taskResult.getRuleId(), taskResults);
        }
    }
    TaskCheckResultResponse taskCheckResultResponse = new TaskCheckResultResponse(taskInDb, singleRuleDataSourceMap, customRuleDataSourceMap, multiRuleDataSourceMap, fileRuleDataSourceMap, allResultMap);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_TASK_DETAIL}", taskCheckResultResponse);
}