Examples with TaskResult com.webank.wedatasphere.qualitis.entity.TaskResult used on opensource projects

Example 6 with TaskResult

use of com.webank.wedatasphere.qualitis.entity.TaskResult in project Qualitis by WeBankFinTech.

the class RuleMetricServiceImpl method getResultsByRuleMetric.

@Override
public DataInfo<RuleMetricValueResponse> getResultsByRuleMetric(long ruleMetricId, int page, int size) throws UnExpectedRequestException {
    DataInfo<RuleMetricValueResponse> dataInfo = new DataInfo<>();
    int total = taskResultDao.countValuesByRuleMetric(ruleMetricId);
    List<TaskResult> values = taskResultDao.findValuesByRuleMetric(ruleMetricId, page, size);
    List<RuleMetricValueResponse> responses = new ArrayList<>(values.size());
    for (TaskResult taskResult : values) {
        if (taskResult == null) {
            continue;
        }
        RuleMetricValueResponse ruleMetricValueResponse = new RuleMetricValueResponse();
        ruleMetricValueResponse.setGenerateTime(taskResult.getCreateTime());
        Rule currentRule = ruleDao.findById(taskResult.getRuleId());
        ruleMetricValueResponse.setRelatedRuleName(currentRule == null ? "Deleted" : currentRule.getName());
        ruleMetricValueResponse.setRuleMetricValue(StringUtils.isBlank(taskResult.getValue()) ? "0" : taskResult.getValue());
        responses.add(ruleMetricValueResponse);
    }
    dataInfo.setContent(responses);
    dataInfo.setTotalCount(total);
    return dataInfo;
}

Example 7 with TaskResult

use of com.webank.wedatasphere.qualitis.entity.TaskResult in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method commonExecution.

@Override
public ApplicationTaskSimpleResponse commonExecution(List<Rule> rules, StringBuffer partition, String executionUser, String nodeName, String startupParam, String clusterName, String setFlag, Map<String, String> execParams, Application newApplication, Date date, StringBuffer runDate) throws RuleVariableNotSupportException, JobSubmitException, RuleVariableNotFoundException, ArgumentException, ConvertException, DataQualityTaskException, TaskTypeException, ClusterInfoNotConfigException, SystemConfigException, UnExpectedRequestException, MetaDataAcquireFailedException, IOException, TaskNotExistException, SemanticException, ParseException, DataSourceMoveException, DataSourceOverSizeException, org.apache.hadoop.hive.ql.parse.ParseException, BothNullDatasourceException, RightNullDatasourceException, LeftNullDatasourceException, java.text.ParseException {
    // current user
    String userName = executionUser;
    // Generate database name.
    String database = generateDatabase(userName);
    LOGGER.info("Succeed to generate database_name: {}", database);
    // Save application
    newApplication.setRuleSize(rules.size());
    newApplication.setSavedDb(database);
    Application saveApplication = applicationDao.saveApplication(newApplication);
    List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
    List<Rule> fileRules = new ArrayList<>();
    Map<Long, Map> dataSourceMysqlConnect = new HashMap<>(2);
    for (Iterator<Rule> iterator = rules.iterator(); iterator.hasNext(); ) {
        Rule currentRule = iterator.next();
        if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
            // Replace with execution parameter and parse datasource to save.
            customReSaveDateSource(currentRule, execParams, clusterName, date);
        }
        List<Map<String, String>> mappingCols = new ArrayList<>();
        getMappingCols(currentRule, mappingCols);
        // Check datasource before submit job.
        try {
            checkDatasource(currentRule, userName, partition, mappingCols, nodeName, clusterName, dataSourceMysqlConnect);
        } catch (BothNullDatasourceException e) {
            Task taskInDb = taskDao.save(new Task(newApplication, newApplication.getSubmitTime(), TaskStatusEnum.SUCCEED.getCode()));
            taskInDb.setClusterName(clusterName);
            TaskRuleSimple taskRuleSimple = new TaskRuleSimple(currentRule, taskInDb);
            Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
            Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
            taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
            for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
                taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
            }
            taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
            taskInDb.setTaskDataSources(taskDataSources);
            taskInDb.setTaskRuleSimples(taskRuleSimples);
            TaskResult taskResult = new TaskResult();
            taskResult.setApplicationId(newApplication.getId());
            taskResult.setCreateTime(newApplication.getSubmitTime());
            taskResult.setRuleId(currentRule.getId());
            taskResult.setResultType("Long");
            taskResult.setValue(0 + "");
            taskResultDao.saveTaskResult(taskResult);
            taskDao.save(taskInDb);
            iterator.remove();
            if (!iterator.hasNext()) {
                throw e;
            }
            continue;
        }
        if (currentRule.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())) {
            fileRules.add(currentRule);
            LOGGER.info("Succeed to find file rule. Rule: {}", currentRule.getId() + " " + currentRule.getName());
            iterator.remove();
        }
    }
    String submitTime = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(date);
    // General task.
    if (!rules.isEmpty()) {
        taskSubmitResults.addAll(executionManager.submitApplication(rules, nodeName, submitTime, userName, database, partition, date, saveApplication, clusterName, startupParam, setFlag, execParams, runDate, dataSourceMysqlConnect));
    }
    // Execute file rule task and save task result.
    if (!fileRules.isEmpty()) {
        taskSubmitResults.add(executionManager.executeFileRule(fileRules, submitTime, saveApplication, userName, clusterName, runDate));
    }
    saveApplication.setTotalTaskNum(taskSubmitResults.size());
    LOGGER.info("Succeed to submit application. result: {}", taskSubmitResults);
    Application applicationInDb = applicationDao.saveApplication(saveApplication);
    LOGGER.info("Succeed to save application. application: {}", applicationInDb);
    return new ApplicationTaskSimpleResponse(taskSubmitResults);
}

Example 8 with TaskResult

use of com.webank.wedatasphere.qualitis.entity.TaskResult in project Qualitis by WeBankFinTech.

the class TaskChecker method checkTaskRuleSimplePass.

@Transactional(rollbackFor = { RuntimeException.class, UnExpectedRequestException.class })
private Boolean checkTaskRuleSimplePass(String applicationId, TaskRuleSimple taskRuleSimple) {
    Boolean passFlag = true;
    List<TaskResult> taskResults = taskResultDao.findByApplicationAndRule(applicationId, taskRuleSimple.getRuleId());
    for (TaskResult taskResult : taskResults) {
        List<TaskRuleAlarmConfig> taskRuleAlarmConfigList = taskRuleSimple.getTaskRuleAlarmConfigList();
        Long ruleMetricId = taskResult.getRuleMetricId();
        if (ruleMetricId != null && ruleMetricId.longValue() != -1) {
            taskRuleAlarmConfigList = taskRuleAlarmConfigList.stream().filter(taskRuleAlarmConfig -> taskRuleAlarmConfig.getRuleMetric().getId().equals(ruleMetricId)).collect(Collectors.toList());
        }
        for (TaskRuleAlarmConfig taskRuleAlarmConfig : taskRuleAlarmConfigList) {
            if (PassUtil.notSafe(applicationId, taskRuleSimple.getRuleId(), taskRuleAlarmConfig, taskResult, taskResultDao)) {
                taskRuleAlarmConfig.setStatus(AlarmConfigStatusEnum.PASS.getCode());
            } else {
                passFlag = false;
                taskRuleAlarmConfig.setStatus(AlarmConfigStatusEnum.NOT_PASS.getCode());
                if (taskRuleSimple.getRuleType().equals(RuleTemplateTypeEnum.CUSTOM.getCode())) {
                    if (taskRuleAlarmConfig.getDeleteFailCheckResult() != null && true == taskRuleAlarmConfig.getDeleteFailCheckResult().booleanValue()) {
                        taskResult.setSaveResult(false);
                        taskResultDao.saveTaskResult(taskResult);
                    }
                } else {
                    if (taskRuleSimple.getDeleteFailCheckResult() != null && true == taskRuleSimple.getDeleteFailCheckResult().booleanValue()) {
                        taskResult.setSaveResult(false);
                        taskResultDao.saveTaskResult(taskResult);
                    }
                }
            }
        }
    }
    return passFlag;
}