Search in sources :

Example 16 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class RuleQueryServiceImpl method getProjectsByUserPerm.

private List<RuleQueryProject> getProjectsByUserPerm(DataSourceQo param, List<ProjectUser> projectUsers, Map<Long, RuleQueryProject> projectMap) {
    if (projectUsers == null || projectUsers.isEmpty()) {
        return null;
    }
    for (ProjectUser projectUser : projectUsers) {
        List<RuleDataSource> projectDataSources = ruleDataSourceDao.findByProjectUser(projectUser.getProject().getId(), param.getCluster(), param.getDb(), param.getTable());
        if (projectDataSources == null || projectDataSources.isEmpty()) {
            continue;
        }
        addRuleDataSource(projectDataSources, projectMap, projectUser.getProject());
    }
    return new ArrayList<>(projectMap.values());
}
Also used : ProjectUser(com.webank.wedatasphere.qualitis.project.entity.ProjectUser) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) ArrayList(java.util.ArrayList)

Example 17 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method commonExecution.

@Override
public ApplicationTaskSimpleResponse commonExecution(List<Rule> rules, StringBuffer partition, String executionUser, String nodeName, String startupParam, String clusterName, String setFlag, Map<String, String> execParams, Application newApplication, Date date, StringBuffer runDate) throws RuleVariableNotSupportException, JobSubmitException, RuleVariableNotFoundException, ArgumentException, ConvertException, DataQualityTaskException, TaskTypeException, ClusterInfoNotConfigException, SystemConfigException, UnExpectedRequestException, MetaDataAcquireFailedException, IOException, TaskNotExistException, SemanticException, ParseException, DataSourceMoveException, DataSourceOverSizeException, org.apache.hadoop.hive.ql.parse.ParseException, BothNullDatasourceException, RightNullDatasourceException, LeftNullDatasourceException, java.text.ParseException {
    // current user
    String userName = executionUser;
    // Generate database name.
    String database = generateDatabase(userName);
    LOGGER.info("Succeed to generate database_name: {}", database);
    // Save application
    newApplication.setRuleSize(rules.size());
    newApplication.setSavedDb(database);
    Application saveApplication = applicationDao.saveApplication(newApplication);
    List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
    List<Rule> fileRules = new ArrayList<>();
    Map<Long, Map> dataSourceMysqlConnect = new HashMap<>(2);
    for (Iterator<Rule> iterator = rules.iterator(); iterator.hasNext(); ) {
        Rule currentRule = iterator.next();
        if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
            // Replace with execution parameter and parse datasource to save.
            customReSaveDateSource(currentRule, execParams, clusterName, date);
        }
        List<Map<String, String>> mappingCols = new ArrayList<>();
        getMappingCols(currentRule, mappingCols);
        // Check datasource before submit job.
        try {
            checkDatasource(currentRule, userName, partition, mappingCols, nodeName, clusterName, dataSourceMysqlConnect);
        } catch (BothNullDatasourceException e) {
            Task taskInDb = taskDao.save(new Task(newApplication, newApplication.getSubmitTime(), TaskStatusEnum.SUCCEED.getCode()));
            taskInDb.setClusterName(clusterName);
            TaskRuleSimple taskRuleSimple = new TaskRuleSimple(currentRule, taskInDb);
            Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
            Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
            taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
            for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
                taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
            }
            taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
            taskInDb.setTaskDataSources(taskDataSources);
            taskInDb.setTaskRuleSimples(taskRuleSimples);
            TaskResult taskResult = new TaskResult();
            taskResult.setApplicationId(newApplication.getId());
            taskResult.setCreateTime(newApplication.getSubmitTime());
            taskResult.setRuleId(currentRule.getId());
            taskResult.setResultType("Long");
            taskResult.setValue(0 + "");
            taskResultDao.saveTaskResult(taskResult);
            taskDao.save(taskInDb);
            iterator.remove();
            if (!iterator.hasNext()) {
                throw e;
            }
            continue;
        }
        if (currentRule.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())) {
            fileRules.add(currentRule);
            LOGGER.info("Succeed to find file rule. Rule: {}", currentRule.getId() + " " + currentRule.getName());
            iterator.remove();
        }
    }
    String submitTime = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(date);
    // General task.
    if (!rules.isEmpty()) {
        taskSubmitResults.addAll(executionManager.submitApplication(rules, nodeName, submitTime, userName, database, partition, date, saveApplication, clusterName, startupParam, setFlag, execParams, runDate, dataSourceMysqlConnect));
    }
    // Execute file rule task and save task result.
    if (!fileRules.isEmpty()) {
        taskSubmitResults.add(executionManager.executeFileRule(fileRules, submitTime, saveApplication, userName, clusterName, runDate));
    }
    saveApplication.setTotalTaskNum(taskSubmitResults.size());
    LOGGER.info("Succeed to submit application. result: {}", taskSubmitResults);
    Application applicationInDb = applicationDao.saveApplication(saveApplication);
    LOGGER.info("Succeed to save application. application: {}", applicationInDb);
    return new ApplicationTaskSimpleResponse(taskSubmitResults);
}
Also used : Task(com.webank.wedatasphere.qualitis.entity.Task) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) ApplicationTaskSimpleResponse(com.webank.wedatasphere.qualitis.response.ApplicationTaskSimpleResponse) Set(java.util.Set) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Date(java.util.Date) TaskDataSource(com.webank.wedatasphere.qualitis.entity.TaskDataSource) BothNullDatasourceException(com.webank.wedatasphere.qualitis.exception.BothNullDatasourceException) TaskResult(com.webank.wedatasphere.qualitis.entity.TaskResult) Rule(com.webank.wedatasphere.qualitis.rule.entity.Rule) Application(com.webank.wedatasphere.qualitis.entity.Application) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) TaskRuleSimple(com.webank.wedatasphere.qualitis.entity.TaskRuleSimple) TaskSubmitResult(com.webank.wedatasphere.qualitis.bean.TaskSubmitResult)

Example 18 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method checkDatasource.

private void checkDatasource(Rule currentRule, String userName, StringBuffer partition, List<Map<String, String>> mappingCols, String nodeName, String clusterName, Map<Long, Map> dataSourceMysqlConnect) throws UnExpectedRequestException, MetaDataAcquireFailedException, DataSourceOverSizeException, DataSourceMoveException, BothNullDatasourceException, LeftNullDatasourceException, RightNullDatasourceException {
    // For multi source rule to check tables' size before submit.
    List<Double> datasourceSizeList = new ArrayList<>(currentRule.getRuleDataSources().size());
    for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
        Map<String, String> mappingCol = null;
        if (ORIGINAL_INDEX.equals(ruleDataSource.getDatasourceIndex())) {
            continue;
        }
        if (ruleDataSource.getDatasourceIndex() != null && mappingCols.get(ruleDataSource.getDatasourceIndex()).size() > 0) {
            mappingCol = mappingCols.get(ruleDataSource.getDatasourceIndex());
        }
        if (ruleDataSource.getLinkisDataSourceId() != null) {
            LOGGER.info("Start to solve relationship datasource info.");
            checkRdmsSqlMetaInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource, mappingCol);
            GeneralResponse<Map> dataSourceInfoDetail = metaDataClient.getDataSourceInfoDetail(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource.getLinkisDataSourceId(), ruleDataSource.getLinkisDataSourceVersionId());
            GeneralResponse<Map> dataSourceConnectParams = metaDataClient.getDataSourceConnectParams(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource.getLinkisDataSourceId(), ruleDataSource.getLinkisDataSourceVersionId());
            Map connectParamsReal = (Map) dataSourceConnectParams.getData().get("connectParams");
            if (connectParamsReal.size() == 0) {
                throw new UnExpectedRequestException("{&THE_DATASOURCE_IS_NOT_DEPLOYED}");
            }
            Map connectParams = (Map) ((Map) dataSourceInfoDetail.getData().get("info")).get("connectParams");
            String dataType = (String) ((Map) ((Map) dataSourceInfoDetail.getData().get("info")).get("dataSourceType")).get("name");
            connectParams.put("dataType", dataType);
            dataSourceMysqlConnect.put(ruleDataSource.getId(), connectParams);
            continue;
        }
        // Parse filter fields.
        List<String> filterFields = getFilterFields(partition.toString());
        if (StringUtils.isNotBlank(ruleDataSource.getDbName()) && !ruleDataSource.getDbName().equals(RuleConstraintEnum.CUSTOM_DATABASE_PREFIS.getValue())) {
            // Get actual fields info.
            List<ColumnInfoDetail> cols = metaDataClient.getColumnInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), userName);
            if (CollectionUtils.isEmpty(cols)) {
                throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "]. {&RULE_DATASOURCE_BE_MOVED}");
            }
            // Get actual partition fields.
            List<String> partitionFields = cols.stream().filter(ColumnInfoDetail::getPartitionField).map(ColumnInfoDetail::getFieldName).collect(Collectors.toList());
            // Check filter fields.
            boolean partitionTable = CollectionUtils.isNotEmpty(partitionFields);
            if (partitionTable && partition.length() > 0) {
                for (String filter : filterFields) {
                    if (!partitionFields.contains(filter)) {
                        throw new UnExpectedRequestException("Table[" + ruleDataSource.getTableName() + "]. {&THE_CHECK_FIELD_DOES_NOT_EXIST_IN_PARTITIONS}[" + filter + "]");
                    }
                }
                // Check partition size.
                PartitionStatisticsInfo partitionStatisticsInfo = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(partition.toString()), userName);
                String fullSize = partitionStatisticsInfo.getPartitionSize();
                ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
                if (clusterInfo != null && StringUtils.isNotBlank(clusterInfo.getSkipDataSize()) && StringUtils.isNotBlank(fullSize)) {
                    double number = 0;
                    String unit = "B";
                    if (!"0B".equals(fullSize)) {
                        number = Double.parseDouble(fullSize.split(" ")[0]);
                        unit = fullSize.split(" ")[1];
                    }
                    datasourceSizeList.add(number);
                    String[] skipDataSize = clusterInfo.getSkipDataSize().split(" ");
                    double res = UnitTransfer.alarmconfigToTaskResult(number, skipDataSize[1], unit);
                    LOGGER.info("Check datasource[" + fullSize + "] if or not oversize with system config[" + clusterInfo.getSkipDataSize() + "]");
                    if (res > Double.parseDouble(skipDataSize[0])) {
                        throw new DataSourceOverSizeException("Table[" + ruleDataSource.getTableName() + "]. {&TABLE_IS_OVERSIZE_WITH_SYSTEM_CONFIG}:[" + clusterInfo.getSkipDataSize() + "]");
                    }
                }
            } else {
                // Check table size.
                TableStatisticsInfo tableStatisticsInfo = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), userName);
                String fullSize = tableStatisticsInfo.getTableSize();
                if (NULL_TABLE_SIZE.equals(fullSize)) {
                    throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "] {&RULE_DATASOURCE_BE_MOVED}");
                }
                ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
                if (clusterInfo != null && StringUtils.isNotBlank(clusterInfo.getSkipDataSize()) && StringUtils.isNotBlank(fullSize)) {
                    LOGGER.info("Check datasource[" + fullSize + "] if or not oversize with system config[" + clusterInfo.getSkipDataSize() + "]");
                    double number = 0;
                    String unit = "B";
                    if (!"0B".equals(fullSize)) {
                        number = Double.parseDouble(fullSize.split(" ")[0]);
                        unit = fullSize.split(" ")[1];
                    }
                    datasourceSizeList.add(number);
                    String[] skipDataSize = clusterInfo.getSkipDataSize().split(" ");
                    double res = UnitTransfer.alarmconfigToTaskResult(number, skipDataSize[1], unit);
                    if (res > Double.parseDouble(skipDataSize[0])) {
                        throw new DataSourceOverSizeException("Table[" + ruleDataSource.getTableName() + "] is oversize with system config:[" + clusterInfo.getSkipDataSize() + "]");
                    }
                }
                partition.delete(0, partition.length());
            }
            if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
                continue;
            }
            if (!metaDataClient.fieldExist(ruleDataSource.getColName(), cols, mappingCol)) {
                throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "] {&RULE_DATASOURCE_BE_MOVED}");
            }
        } else if (StringUtils.isNotBlank(currentRule.getCsId())) {
            checkDatasourceInContextService(ruleDataSource, mappingCol, clusterName, userName, nodeName, currentRule.getCsId());
        }
    }
    if (CollectionUtils.isNotEmpty(datasourceSizeList) && currentRule.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())) {
        double left = datasourceSizeList.get(0);
        double right = datasourceSizeList.get(1);
        LOGGER.info("Current multi source rule left table size number[{}], right table size number[{}]", left, right);
        if (left == 0 && right == 0) {
            throw new BothNullDatasourceException("{&BOTH_SIDE_ARE_NULL}");
        } else if (left == 0) {
            throw new LeftNullDatasourceException("{&ONE_SIDE_ARE_NULL}");
        } else if (right == 0) {
            throw new RightNullDatasourceException("{&ONE_SIDE_ARE_NULL}");
        }
    }
}
Also used : UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) ArrayList(java.util.ArrayList) LeftNullDatasourceException(com.webank.wedatasphere.qualitis.exception.LeftNullDatasourceException) RightNullDatasourceException(com.webank.wedatasphere.qualitis.exception.RightNullDatasourceException) ColumnInfoDetail(com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail) DataSourceOverSizeException(com.webank.wedatasphere.qualitis.exception.DataSourceOverSizeException) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) DataSourceMoveException(com.webank.wedatasphere.qualitis.exception.DataSourceMoveException) BothNullDatasourceException(com.webank.wedatasphere.qualitis.exception.BothNullDatasourceException) PartitionStatisticsInfo(com.webank.wedatasphere.qualitis.metadata.response.table.PartitionStatisticsInfo) TableStatisticsInfo(com.webank.wedatasphere.qualitis.metadata.response.table.TableStatisticsInfo) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap)

Example 19 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method submitRulesWithDynamicPartition.

private void submitRulesWithDynamicPartition(List<ApplicationSubmitRequest> applicationSubmitRequests, Long projectId, Long ruleGroupId, List<Rule> rules, List<Long> ruleIds, String executionUser, boolean dynamicPartition, String clusterName, StringBuffer partition, String dynamicPartitionPrefix) throws UnExpectedRequestException, NoPartitionException {
    if (dynamicPartition) {
        for (Rule rule : rules) {
            RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
            if (ruleDataSource == null) {
                throw new UnExpectedRequestException("Rule datasource has been broken.");
            }
            if (RuleTypeEnum.CUSTOM_RULE.getCode().equals(rule.getRuleType())) {
                throw new UnExpectedRequestException(RuleTypeEnum.CUSTOM_RULE.getMessage() + " {&IS_NOT_SUPPORT}");
            }
            List<Map> currentPartitionMap;
            try {
                TableStatisticsInfo tableStatisticsInfo = metaDataClient.getTableStatisticsInfo(StringUtils.isBlank(clusterName) ? ruleDataSource.getClusterName() : clusterName, ruleDataSource.getDbName(), ruleDataSource.getTableName(), executionUser);
                currentPartitionMap = tableStatisticsInfo.getPartitions();
                if (StringUtils.isNotEmpty(dynamicPartitionPrefix)) {
                    metaDataClient.getPartitionStatisticsInfo(StringUtils.isBlank(clusterName) ? ruleDataSource.getClusterName() : clusterName, ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(dynamicPartitionPrefix), executionUser);
                }
            } catch (MetaDataAcquireFailedException e) {
                LOGGER.error("Dynamic submit failed.", e);
                throw new NoPartitionException(ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + "{&HAS_NO_PARTITIONS_TO_BE_EXECUTE}");
            }
            List<String> partitionList = new ArrayList<>();
            if (CollectionUtils.isNotEmpty(currentPartitionMap)) {
                ruleIds.remove(rule.getId());
                getPartitionListAndSubmit(currentPartitionMap, partitionList, rule, partition, filterToPartitionPath(dynamicPartitionPrefix), applicationSubmitRequests, projectId, ruleGroupId);
            } else {
                throw new NoPartitionException(ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + "{&HAS_NO_PARTITIONS_TO_BE_EXECUTE}");
            }
        }
    }
}
Also used : UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) MetaDataAcquireFailedException(com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException) NoPartitionException(com.webank.wedatasphere.qualitis.exception.NoPartitionException) ArrayList(java.util.ArrayList) Rule(com.webank.wedatasphere.qualitis.rule.entity.Rule) TableStatisticsInfo(com.webank.wedatasphere.qualitis.metadata.response.table.TableStatisticsInfo) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap)

Example 20 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method dataSourceExecution.

@Override
public GeneralResponse<?> dataSourceExecution(DataSourceExecutionRequest request) throws UnExpectedRequestException, PermissionDeniedRequestException {
    LOGGER.info("Execute application by datasource. cluster: {}, database: {}, table: {}", request.getCluster(), request.getDatabase(), request.getTable());
    // Check Arguments.
    DataSourceExecutionRequest.checkRequest(request);
    String loginUser = getLoginUser(httpServletRequest, request.getCreateUser(), request.getAsync());
    // Find all rule datasources by user.
    List<RuleDataSource> ruleDataSources = new ArrayList<>();
    ruleDataSources.addAll(ruleDataSourceDao.findDatasourcesByUser(loginUser, request.getCluster(), request.getDatabase(), request.getTable()));
    List<Rule> rules = ruleDataSources.stream().filter(r -> r.getClusterName().equals(request.getCluster()) && r.getDbName().equals(request.getDatabase()) && r.getTableName().equals(request.getTable())).map(RuleDataSource::getRule).distinct().filter(rule -> {
        if (!request.getCrossTable()) {
            return rule.getRuleDataSources().size() == 1;
        }
        return true;
    }).collect(Collectors.toList());
    if (CollectionUtils.isEmpty(rules)) {
        throw new UnExpectedRequestException("{&NO_RULE_CAN_BE_EXECUTED}");
    }
    checkPermissionCreateUserProxyExecuteUser(request.getCreateUser(), request.getExecutionUser());
    ApplicationProjectResponse applicationProjectResponse = new ApplicationProjectResponse();
    List<ApplicationSubmitRequest> applicationSubmitRequests = new ArrayList<>();
    StringBuffer partition = new StringBuffer();
    StringBuffer runDate = new StringBuffer();
    Map<String, String> execParamMap = new HashMap<>(5);
    parseExecParams(partition, runDate, request.getExecutionParam(), execParamMap);
    List<Project> projects = rules.stream().map(Rule::getProject).distinct().collect(Collectors.toList());
    for (Project projectInDb : projects) {
        // Check permissions of project
        List<Integer> permissions = new ArrayList<>();
        permissions.add(ProjectUserPermissionEnum.OPERATOR.getCode());
        projectService.checkProjectPermission(projectInDb, loginUser, permissions);
        List<Rule> currentRules = rules.stream().filter(rule -> rule.getProject().getId().equals(projectInDb.getId())).collect(Collectors.toList());
        List<RuleGroup> currentRuleGroups = currentRules.stream().map(Rule::getRuleGroup).distinct().collect(Collectors.toList());
        for (RuleGroup ruleGroup : currentRuleGroups) {
            List<Rule> currentRulesOfGroup = currentRules.stream().filter(rule -> rule.getRuleGroup().getRuleGroupName().equals(ruleGroup.getRuleGroupName())).distinct().collect(Collectors.toList());
            List<Long> currentRuleIds = currentRulesOfGroup.stream().map(Rule::getId).distinct().collect(Collectors.toList());
            LOGGER.info("Succeed to find current rules of one group with datasource. rule_id: {}", currentRuleIds);
            // Dynamic partition.
            try {
                submitRulesWithDynamicPartition(applicationSubmitRequests, projectInDb.getId(), ruleGroup.getId(), rules, currentRuleIds, request.getExecutionUser(), request.getDyNamicPartition(), request.getClusterName(), partition, request.getDyNamicPartitionPrefix());
            } catch (ResourceAccessException e) {
                // Record submit failed applicatoin.
                generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
                LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
            } catch (NoPartitionException e) {
                // Record submit failed applicatoin.
                generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
                LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
            } catch (Exception e) {
                // Record submit failed applicatoin.
                generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.UNKNOWN_ERROR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
                LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
            }
            if (CollectionUtils.isNotEmpty(currentRuleIds)) {
                applicationSubmitRequests.add(new ApplicationSubmitRequest(request.getJobId(), projectInDb.getId(), ruleGroup.getId(), currentRuleIds, partition));
            }
        }
        projectEventService.record(projectInDb.getId(), loginUser, "submit datasource execution", "rule name[" + Arrays.toString(rules.stream().filter(rule -> rule.getProject().getId().equals(projectInDb.getId())).toArray()) + "].", EventTypeEnum.SUBMIT_PROJECT.getCode());
    }
    GeneralResponse<ApplicationTaskSimpleResponse> generalResponse;
    for (ApplicationSubmitRequest applicationSubmitRequest : applicationSubmitRequests) {
        if (StringUtils.isNotBlank(request.getJobId())) {
            applicationSubmitRequest.setJobId(request.getJobId());
        }
        generalResponse = (GeneralResponse<ApplicationTaskSimpleResponse>) outerExecutionService.submitRules(applicationSubmitRequest.getRuleIds(), applicationSubmitRequest.getPartition(), loginUser, request.getExecutionUser(), DEFAULT_NODE_NAME, applicationSubmitRequest.getProjectId(), applicationSubmitRequest.getRuleGroupId(), request.getStartupParamName(), request.getClusterName(), request.getSetFlag(), execParamMap, request.getExecutionParam(), runDate, InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode());
        applicationProjectResponse.getApplicationTaskSimpleResponses().add(generalResponse.getData());
    }
    return new GeneralResponse<>("200", "{&SUCCEED_TO_DISPATCH_TASK}", applicationProjectResponse);
}
Also used : AlarmConfigStatusEnum(com.webank.wedatasphere.qualitis.constant.AlarmConfigStatusEnum) Arrays(java.util.Arrays) StringUtils(org.apache.commons.lang.StringUtils) ClusterInfoNotConfigException(com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException) Autowired(org.springframework.beans.factory.annotation.Autowired) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) AlarmConfig(com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig) ApplicationTaskSimpleResponse(com.webank.wedatasphere.qualitis.response.ApplicationTaskSimpleResponse) DataSourceOverSizeException(com.webank.wedatasphere.qualitis.exception.DataSourceOverSizeException) GroupExecutionRequest(com.webank.wedatasphere.qualitis.request.GroupExecutionRequest) HttpUtils(com.webank.wedatasphere.qualitis.util.HttpUtils) Map(java.util.Map) LocaleParser(com.webank.wedatasphere.qualitis.parser.LocaleParser) JobSubmitException(com.webank.wedatasphere.qualitis.exception.JobSubmitException) ConvertException(com.webank.wedatasphere.qualitis.exception.ConvertException) MonitorManager(com.webank.wedatasphere.qualitis.job.MonitorManager) ProjectUserPermissionEnum(com.webank.wedatasphere.qualitis.project.constant.ProjectUserPermissionEnum) Set(java.util.Set) RuleConstraintEnum(com.webank.wedatasphere.qualitis.metadata.constant.RuleConstraintEnum) UserProxyUser(com.webank.wedatasphere.qualitis.entity.UserProxyUser) TableStat(com.alibaba.druid.stat.TableStat) InvokeTypeEnum(com.webank.wedatasphere.qualitis.constant.InvokeTypeEnum) LeftNullDatasourceException(com.webank.wedatasphere.qualitis.exception.LeftNullDatasourceException) MetaDataClient(com.webank.wedatasphere.qualitis.metadata.client.MetaDataClient) DbType(com.alibaba.druid.DbType) GeneralExecutionRequest(com.webank.wedatasphere.qualitis.request.GeneralExecutionRequest) Project(com.webank.wedatasphere.qualitis.project.entity.Project) TaskStatusEnum(com.webank.wedatasphere.qualitis.constant.TaskStatusEnum) JobKillException(com.webank.wedatasphere.qualitis.exception.JobKillException) RuleDataSourceDao(com.webank.wedatasphere.qualitis.rule.dao.RuleDataSourceDao) Rule(com.webank.wedatasphere.qualitis.rule.entity.Rule) ProjectDao(com.webank.wedatasphere.qualitis.project.dao.ProjectDao) ApplicationTaskResponse(com.webank.wedatasphere.qualitis.response.ApplicationTaskResponse) ArrayList(java.util.ArrayList) LinkedHashMap(java.util.LinkedHashMap) HttpServletRequest(javax.servlet.http.HttpServletRequest) RuleListExecutionRequest(com.webank.wedatasphere.qualitis.request.RuleListExecutionRequest) TaskDao(com.webank.wedatasphere.qualitis.dao.TaskDao) ApplicationResultResponse(com.webank.wedatasphere.qualitis.response.ApplicationResultResponse) Service(org.springframework.stereotype.Service) TaskTypeException(com.webank.wedatasphere.qualitis.exception.TaskTypeException) TaskResult(com.webank.wedatasphere.qualitis.entity.TaskResult) ArgumentException(com.webank.wedatasphere.qualitis.exception.ArgumentException) GetTaskLogRequest(com.webank.wedatasphere.qualitis.request.GetTaskLogRequest) CsTableInfoDetail(com.webank.wedatasphere.qualitis.metadata.response.table.CsTableInfoDetail) MySqlSchemaStatVisitor(com.alibaba.druid.sql.dialect.mysql.visitor.MySqlSchemaStatVisitor) SystemConfigDao(com.webank.wedatasphere.qualitis.dao.SystemConfigDao) ExceptionUtils(org.apache.commons.lang.exception.ExceptionUtils) TaskRuleSimpleRepository(com.webank.wedatasphere.qualitis.dao.repository.TaskRuleSimpleRepository) BothNullDatasourceException(com.webank.wedatasphere.qualitis.exception.BothNullDatasourceException) IOException(java.io.IOException) TaskRuleSimple(com.webank.wedatasphere.qualitis.entity.TaskRuleSimple) DataInfo(com.webank.wedatasphere.qualitis.metadata.response.DataInfo) SystemKeyConfig(com.webank.wedatasphere.qualitis.config.SystemKeyConfig) SQLStatement(com.alibaba.druid.sql.ast.SQLStatement) UserDao(com.webank.wedatasphere.qualitis.dao.UserDao) UnitTransfer(com.webank.wedatasphere.qualitis.rule.util.UnitTransfer) LogResult(com.webank.wedatasphere.qualitis.bean.LogResult) TaskDataSource(com.webank.wedatasphere.qualitis.entity.TaskDataSource) Date(java.util.Date) LinkisConfig(com.webank.wedatasphere.qualitis.config.LinkisConfig) LoggerFactory(org.slf4j.LoggerFactory) GetUserColumnByCsRequest(com.webank.wedatasphere.qualitis.metadata.request.GetUserColumnByCsRequest) Application(com.webank.wedatasphere.qualitis.entity.Application) TaskNotExistException(com.webank.wedatasphere.qualitis.exception.TaskNotExistException) SemanticException(org.apache.hadoop.hive.ql.parse.SemanticException) ProjectUserDao(com.webank.wedatasphere.qualitis.project.dao.ProjectUserDao) DataSourceExecutionRequest(com.webank.wedatasphere.qualitis.request.DataSourceExecutionRequest) PermissionDeniedRequestException(com.webank.wedatasphere.qualitis.exception.PermissionDeniedRequestException) ApplicationCommentEnum(com.webank.wedatasphere.qualitis.constant.ApplicationCommentEnum) EventTypeEnum(com.webank.wedatasphere.qualitis.project.constant.EventTypeEnum) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) DateExprReplaceUtil(com.webank.wedatasphere.qualitis.util.DateExprReplaceUtil) ClusterInfoDao(com.webank.wedatasphere.qualitis.dao.ClusterInfoDao) ParseException(org.apache.hadoop.hive.ql.parse.ParseException) Task(com.webank.wedatasphere.qualitis.entity.Task) OuterExecutionService(com.webank.wedatasphere.qualitis.service.OuterExecutionService) RuleGroupDao(com.webank.wedatasphere.qualitis.rule.dao.RuleGroupDao) TaskRuleAlarmConfig(com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig) SystemConfigException(com.webank.wedatasphere.qualitis.exception.SystemConfigException) ProjectService(com.webank.wedatasphere.qualitis.project.service.ProjectService) RuleVariableNotSupportException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotSupportException) ProjectEventService(com.webank.wedatasphere.qualitis.project.service.ProjectEventService) ExecutionManager(com.webank.wedatasphere.qualitis.submitter.ExecutionManager) PartitionStatisticsInfo(com.webank.wedatasphere.qualitis.metadata.response.table.PartitionStatisticsInfo) Context(javax.ws.rs.core.Context) UuidGenerator(com.webank.wedatasphere.qualitis.util.UuidGenerator) Collectors(java.util.stream.Collectors) RuleDataSourceMapping(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSourceMapping) MetaDataAcquireFailedException(com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException) ApplicationProjectResponse(com.webank.wedatasphere.qualitis.response.ApplicationProjectResponse) ExecutionManagerImpl(com.webank.wedatasphere.qualitis.submitter.impl.ExecutionManagerImpl) List(java.util.List) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) JdbcConstants(com.alibaba.druid.util.JdbcConstants) ApplicationSubmitRequest(com.webank.wedatasphere.qualitis.request.ApplicationSubmitRequest) SystemConfig(com.webank.wedatasphere.qualitis.entity.SystemConfig) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) LogPartialException(com.webank.wedatasphere.qualitis.exception.LogPartialException) BeanUtils(org.springframework.beans.BeanUtils) RuleGroup(com.webank.wedatasphere.qualitis.rule.entity.RuleGroup) SpecCharEnum(com.webank.wedatasphere.qualitis.constant.SpecCharEnum) HashMap(java.util.HashMap) HashSet(java.util.HashSet) User(com.webank.wedatasphere.qualitis.entity.User) CollectionUtils(org.apache.commons.collections.CollectionUtils) Propagation(org.springframework.transaction.annotation.Propagation) Name(com.alibaba.druid.stat.TableStat.Name) TaskSubmitResult(com.webank.wedatasphere.qualitis.bean.TaskSubmitResult) DataQualityTaskException(com.webank.wedatasphere.qualitis.exception.DataQualityTaskException) ApplicationStatusEnum(com.webank.wedatasphere.qualitis.constant.ApplicationStatusEnum) TaskDataSourceRepository(com.webank.wedatasphere.qualitis.dao.repository.TaskDataSourceRepository) GetUserTableByCsIdRequest(com.webank.wedatasphere.qualitis.metadata.request.GetUserTableByCsIdRequest) RuleVariableNotFoundException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotFoundException) DataSourceMoveException(com.webank.wedatasphere.qualitis.exception.DataSourceMoveException) SQLUtils(com.alibaba.druid.sql.SQLUtils) Logger(org.slf4j.Logger) Iterator(java.util.Iterator) RuleMetric(com.webank.wedatasphere.qualitis.entity.RuleMetric) ResourceAccessException(org.springframework.web.client.ResourceAccessException) ColumnInfoDetail(com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail) FastDateFormat(org.apache.commons.lang3.time.FastDateFormat) TableStatisticsInfo(com.webank.wedatasphere.qualitis.metadata.response.table.TableStatisticsInfo) RuleTypeEnum(com.webank.wedatasphere.qualitis.rule.constant.RuleTypeEnum) ProjectExecutionRequest(com.webank.wedatasphere.qualitis.request.ProjectExecutionRequest) RightNullDatasourceException(com.webank.wedatasphere.qualitis.exception.RightNullDatasourceException) RuleDao(com.webank.wedatasphere.qualitis.rule.dao.RuleDao) ApplicationDao(com.webank.wedatasphere.qualitis.dao.ApplicationDao) NoPartitionException(com.webank.wedatasphere.qualitis.exception.NoPartitionException) TaskResultDao(com.webank.wedatasphere.qualitis.dao.TaskResultDao) HiveSqlParser(com.webank.wedatasphere.qualitis.parser.HiveSqlParser) Transactional(org.springframework.transaction.annotation.Transactional) ApplicationSubmitRequest(com.webank.wedatasphere.qualitis.request.ApplicationSubmitRequest) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) ApplicationTaskSimpleResponse(com.webank.wedatasphere.qualitis.response.ApplicationTaskSimpleResponse) ApplicationProjectResponse(com.webank.wedatasphere.qualitis.response.ApplicationProjectResponse) RuleGroup(com.webank.wedatasphere.qualitis.rule.entity.RuleGroup) ResourceAccessException(org.springframework.web.client.ResourceAccessException) Date(java.util.Date) ClusterInfoNotConfigException(com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException) DataSourceOverSizeException(com.webank.wedatasphere.qualitis.exception.DataSourceOverSizeException) JobSubmitException(com.webank.wedatasphere.qualitis.exception.JobSubmitException) ConvertException(com.webank.wedatasphere.qualitis.exception.ConvertException) LeftNullDatasourceException(com.webank.wedatasphere.qualitis.exception.LeftNullDatasourceException) JobKillException(com.webank.wedatasphere.qualitis.exception.JobKillException) TaskTypeException(com.webank.wedatasphere.qualitis.exception.TaskTypeException) ArgumentException(com.webank.wedatasphere.qualitis.exception.ArgumentException) BothNullDatasourceException(com.webank.wedatasphere.qualitis.exception.BothNullDatasourceException) IOException(java.io.IOException) TaskNotExistException(com.webank.wedatasphere.qualitis.exception.TaskNotExistException) SemanticException(org.apache.hadoop.hive.ql.parse.SemanticException) PermissionDeniedRequestException(com.webank.wedatasphere.qualitis.exception.PermissionDeniedRequestException) ParseException(org.apache.hadoop.hive.ql.parse.ParseException) SystemConfigException(com.webank.wedatasphere.qualitis.exception.SystemConfigException) RuleVariableNotSupportException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotSupportException) MetaDataAcquireFailedException(com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) LogPartialException(com.webank.wedatasphere.qualitis.exception.LogPartialException) DataQualityTaskException(com.webank.wedatasphere.qualitis.exception.DataQualityTaskException) RuleVariableNotFoundException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotFoundException) DataSourceMoveException(com.webank.wedatasphere.qualitis.exception.DataSourceMoveException) ResourceAccessException(org.springframework.web.client.ResourceAccessException) RightNullDatasourceException(com.webank.wedatasphere.qualitis.exception.RightNullDatasourceException) NoPartitionException(com.webank.wedatasphere.qualitis.exception.NoPartitionException) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) Project(com.webank.wedatasphere.qualitis.project.entity.Project) NoPartitionException(com.webank.wedatasphere.qualitis.exception.NoPartitionException) Rule(com.webank.wedatasphere.qualitis.rule.entity.Rule)

Aggregations

RuleDataSource (com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource)37 ArrayList (java.util.ArrayList)29 Rule (com.webank.wedatasphere.qualitis.rule.entity.Rule)21 AlarmConfig (com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig)18 UnExpectedRequestException (com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException)14 Template (com.webank.wedatasphere.qualitis.rule.entity.Template)11 HashSet (java.util.HashSet)11 List (java.util.List)10 Set (java.util.Set)10 Transactional (org.springframework.transaction.annotation.Transactional)10 RuleMetric (com.webank.wedatasphere.qualitis.entity.RuleMetric)9 Project (com.webank.wedatasphere.qualitis.project.entity.Project)9 GeneralResponse (com.webank.wedatasphere.qualitis.response.GeneralResponse)9 RuleDataSourceMapping (com.webank.wedatasphere.qualitis.rule.entity.RuleDataSourceMapping)9 Date (java.util.Date)9 HashMap (java.util.HashMap)9 RuleVariable (com.webank.wedatasphere.qualitis.rule.entity.RuleVariable)8 Map (java.util.Map)8 SpecCharEnum (com.webank.wedatasphere.qualitis.constant.SpecCharEnum)6 MetaDataAcquireFailedException (com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException)6