Examples with BothNullDatasourceException com.webank.wedatasphere.qualitis.exception.BothNullDatasourceException used on opensource projects

Example 1 with BothNullDatasourceException

use of com.webank.wedatasphere.qualitis.exception.BothNullDatasourceException in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method commonExecution.

@Override
public ApplicationTaskSimpleResponse commonExecution(List<Rule> rules, StringBuffer partition, String executionUser, String nodeName, String startupParam, String clusterName, String setFlag, Map<String, String> execParams, Application newApplication, Date date, StringBuffer runDate) throws RuleVariableNotSupportException, JobSubmitException, RuleVariableNotFoundException, ArgumentException, ConvertException, DataQualityTaskException, TaskTypeException, ClusterInfoNotConfigException, SystemConfigException, UnExpectedRequestException, MetaDataAcquireFailedException, IOException, TaskNotExistException, SemanticException, ParseException, DataSourceMoveException, DataSourceOverSizeException, org.apache.hadoop.hive.ql.parse.ParseException, BothNullDatasourceException, RightNullDatasourceException, LeftNullDatasourceException, java.text.ParseException {
    // current user
    String userName = executionUser;
    // Generate database name.
    String database = generateDatabase(userName);
    LOGGER.info("Succeed to generate database_name: {}", database);
    // Save application
    newApplication.setRuleSize(rules.size());
    newApplication.setSavedDb(database);
    Application saveApplication = applicationDao.saveApplication(newApplication);
    List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
    List<Rule> fileRules = new ArrayList<>();
    Map<Long, Map> dataSourceMysqlConnect = new HashMap<>(2);
    for (Iterator<Rule> iterator = rules.iterator(); iterator.hasNext(); ) {
        Rule currentRule = iterator.next();
        if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
            // Replace with execution parameter and parse datasource to save.
            customReSaveDateSource(currentRule, execParams, clusterName, date);
        }
        List<Map<String, String>> mappingCols = new ArrayList<>();
        getMappingCols(currentRule, mappingCols);
        // Check datasource before submit job.
        try {
            checkDatasource(currentRule, userName, partition, mappingCols, nodeName, clusterName, dataSourceMysqlConnect);
        } catch (BothNullDatasourceException e) {
            Task taskInDb = taskDao.save(new Task(newApplication, newApplication.getSubmitTime(), TaskStatusEnum.SUCCEED.getCode()));
            taskInDb.setClusterName(clusterName);
            TaskRuleSimple taskRuleSimple = new TaskRuleSimple(currentRule, taskInDb);
            Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
            Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
            taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
            for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
                taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
            }
            taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
            taskInDb.setTaskDataSources(taskDataSources);
            taskInDb.setTaskRuleSimples(taskRuleSimples);
            TaskResult taskResult = new TaskResult();
            taskResult.setApplicationId(newApplication.getId());
            taskResult.setCreateTime(newApplication.getSubmitTime());
            taskResult.setRuleId(currentRule.getId());
            taskResult.setResultType("Long");
            taskResult.setValue(0 + "");
            taskResultDao.saveTaskResult(taskResult);
            taskDao.save(taskInDb);
            iterator.remove();
            if (!iterator.hasNext()) {
                throw e;
            }
            continue;
        }
        if (currentRule.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())) {
            fileRules.add(currentRule);
            LOGGER.info("Succeed to find file rule. Rule: {}", currentRule.getId() + " " + currentRule.getName());
            iterator.remove();
        }
    }
    String submitTime = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(date);
    // General task.
    if (!rules.isEmpty()) {
        taskSubmitResults.addAll(executionManager.submitApplication(rules, nodeName, submitTime, userName, database, partition, date, saveApplication, clusterName, startupParam, setFlag, execParams, runDate, dataSourceMysqlConnect));
    }
    // Execute file rule task and save task result.
    if (!fileRules.isEmpty()) {
        taskSubmitResults.add(executionManager.executeFileRule(fileRules, submitTime, saveApplication, userName, clusterName, runDate));
    }
    saveApplication.setTotalTaskNum(taskSubmitResults.size());
    LOGGER.info("Succeed to submit application. result: {}", taskSubmitResults);
    Application applicationInDb = applicationDao.saveApplication(saveApplication);
    LOGGER.info("Succeed to save application. application: {}", applicationInDb);
    return new ApplicationTaskSimpleResponse(taskSubmitResults);
}

Example 2 with BothNullDatasourceException

use of com.webank.wedatasphere.qualitis.exception.BothNullDatasourceException in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method checkDatasource.

private void checkDatasource(Rule currentRule, String userName, StringBuffer partition, List<Map<String, String>> mappingCols, String nodeName, String clusterName, Map<Long, Map> dataSourceMysqlConnect) throws UnExpectedRequestException, MetaDataAcquireFailedException, DataSourceOverSizeException, DataSourceMoveException, BothNullDatasourceException, LeftNullDatasourceException, RightNullDatasourceException {
    // For multi source rule to check tables' size before submit.
    List<Double> datasourceSizeList = new ArrayList<>(currentRule.getRuleDataSources().size());
    for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
        Map<String, String> mappingCol = null;
        if (ORIGINAL_INDEX.equals(ruleDataSource.getDatasourceIndex())) {
            continue;
        }
        if (ruleDataSource.getDatasourceIndex() != null && mappingCols.get(ruleDataSource.getDatasourceIndex()).size() > 0) {
            mappingCol = mappingCols.get(ruleDataSource.getDatasourceIndex());
        }
        if (ruleDataSource.getLinkisDataSourceId() != null) {
            LOGGER.info("Start to solve relationship datasource info.");
            checkRdmsSqlMetaInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource, mappingCol);
            GeneralResponse<Map> dataSourceInfoDetail = metaDataClient.getDataSourceInfoDetail(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource.getLinkisDataSourceId(), ruleDataSource.getLinkisDataSourceVersionId());
            GeneralResponse<Map> dataSourceConnectParams = metaDataClient.getDataSourceConnectParams(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource.getLinkisDataSourceId(), ruleDataSource.getLinkisDataSourceVersionId());
            Map connectParamsReal = (Map) dataSourceConnectParams.getData().get("connectParams");
            if (connectParamsReal.size() == 0) {
                throw new UnExpectedRequestException("{&THE_DATASOURCE_IS_NOT_DEPLOYED}");
            }
            Map connectParams = (Map) ((Map) dataSourceInfoDetail.getData().get("info")).get("connectParams");
            String dataType = (String) ((Map) ((Map) dataSourceInfoDetail.getData().get("info")).get("dataSourceType")).get("name");
            connectParams.put("dataType", dataType);
            dataSourceMysqlConnect.put(ruleDataSource.getId(), connectParams);
            continue;
        }
        // Parse filter fields.
        List<String> filterFields = getFilterFields(partition.toString());
        if (StringUtils.isNotBlank(ruleDataSource.getDbName()) && !ruleDataSource.getDbName().equals(RuleConstraintEnum.CUSTOM_DATABASE_PREFIS.getValue())) {
            // Get actual fields info.
            List<ColumnInfoDetail> cols = metaDataClient.getColumnInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), userName);
            if (CollectionUtils.isEmpty(cols)) {
                throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "]. {&RULE_DATASOURCE_BE_MOVED}");
            }
            // Get actual partition fields.
            List<String> partitionFields = cols.stream().filter(ColumnInfoDetail::getPartitionField).map(ColumnInfoDetail::getFieldName).collect(Collectors.toList());
            // Check filter fields.
            boolean partitionTable = CollectionUtils.isNotEmpty(partitionFields);
            if (partitionTable && partition.length() > 0) {
                for (String filter : filterFields) {
                    if (!partitionFields.contains(filter)) {
                        throw new UnExpectedRequestException("Table[" + ruleDataSource.getTableName() + "]. {&THE_CHECK_FIELD_DOES_NOT_EXIST_IN_PARTITIONS}[" + filter + "]");
                    }
                }
                // Check partition size.
                PartitionStatisticsInfo partitionStatisticsInfo = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(partition.toString()), userName);
                String fullSize = partitionStatisticsInfo.getPartitionSize();
                ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
                if (clusterInfo != null && StringUtils.isNotBlank(clusterInfo.getSkipDataSize()) && StringUtils.isNotBlank(fullSize)) {
                    double number = 0;
                    String unit = "B";
                    if (!"0B".equals(fullSize)) {
                        number = Double.parseDouble(fullSize.split(" ")[0]);
                        unit = fullSize.split(" ")[1];
                    }
                    datasourceSizeList.add(number);
                    String[] skipDataSize = clusterInfo.getSkipDataSize().split(" ");
                    double res = UnitTransfer.alarmconfigToTaskResult(number, skipDataSize[1], unit);
                    LOGGER.info("Check datasource[" + fullSize + "] if or not oversize with system config[" + clusterInfo.getSkipDataSize() + "]");
                    if (res > Double.parseDouble(skipDataSize[0])) {
                        throw new DataSourceOverSizeException("Table[" + ruleDataSource.getTableName() + "]. {&TABLE_IS_OVERSIZE_WITH_SYSTEM_CONFIG}:[" + clusterInfo.getSkipDataSize() + "]");
                    }
                }
            } else {
                // Check table size.
                TableStatisticsInfo tableStatisticsInfo = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), userName);
                String fullSize = tableStatisticsInfo.getTableSize();
                if (NULL_TABLE_SIZE.equals(fullSize)) {
                    throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "] {&RULE_DATASOURCE_BE_MOVED}");
                }
                ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
                if (clusterInfo != null && StringUtils.isNotBlank(clusterInfo.getSkipDataSize()) && StringUtils.isNotBlank(fullSize)) {
                    LOGGER.info("Check datasource[" + fullSize + "] if or not oversize with system config[" + clusterInfo.getSkipDataSize() + "]");
                    double number = 0;
                    String unit = "B";
                    if (!"0B".equals(fullSize)) {
                        number = Double.parseDouble(fullSize.split(" ")[0]);
                        unit = fullSize.split(" ")[1];
                    }
                    datasourceSizeList.add(number);
                    String[] skipDataSize = clusterInfo.getSkipDataSize().split(" ");
                    double res = UnitTransfer.alarmconfigToTaskResult(number, skipDataSize[1], unit);
                    if (res > Double.parseDouble(skipDataSize[0])) {
                        throw new DataSourceOverSizeException("Table[" + ruleDataSource.getTableName() + "] is oversize with system config:[" + clusterInfo.getSkipDataSize() + "]");
                    }
                }
                partition.delete(0, partition.length());
            }
            if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
                continue;
            }
            if (!metaDataClient.fieldExist(ruleDataSource.getColName(), cols, mappingCol)) {
                throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "] {&RULE_DATASOURCE_BE_MOVED}");
            }
        } else if (StringUtils.isNotBlank(currentRule.getCsId())) {
            checkDatasourceInContextService(ruleDataSource, mappingCol, clusterName, userName, nodeName, currentRule.getCsId());
        }
    }
    if (CollectionUtils.isNotEmpty(datasourceSizeList) && currentRule.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())) {
        double left = datasourceSizeList.get(0);
        double right = datasourceSizeList.get(1);
        LOGGER.info("Current multi source rule left table size number[{}], right table size number[{}]", left, right);
        if (left == 0 && right == 0) {
            throw new BothNullDatasourceException("{&BOTH_SIDE_ARE_NULL}");
        } else if (left == 0) {
            throw new LeftNullDatasourceException("{&ONE_SIDE_ARE_NULL}");
        } else if (right == 0) {
            throw new RightNullDatasourceException("{&ONE_SIDE_ARE_NULL}");
        }
    }
}

Example 3 with BothNullDatasourceException

use of com.webank.wedatasphere.qualitis.exception.BothNullDatasourceException in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method submitRules.

/**
 * Generate jobs by rules and submit jobs to linkis.
 *
 * @param ruleIds
 * @param partition
 * @param createUser
 * @param executionUser
 * @param nodeName
 * @param projectId
 * @param ruleGroupId
 * @param execParams
 * @param runDate
 * @param invokeCode
 * @return
 * @throws UnExpectedRequestException
 */
@Override
@Transactional(propagation = Propagation.REQUIRES_NEW, rollbackFor = { RuntimeException.class, UnExpectedRequestException.class })
public GeneralResponse<?> submitRules(List<Long> ruleIds, StringBuffer partition, String createUser, String executionUser, String nodeName, Long projectId, Long ruleGroupId, String startupParam, String clusterName, String setFlag, Map<String, String> execParams, String execParamStr, StringBuffer runDate, Integer invokeCode) {
    // Get rule eneity.
    List<Rule> rules = ruleDao.findByIds(ruleIds);
    // Init application basic info.
    Date date = new Date();
    Application newApplication = outerExecutionService.generateApplicationInfo(createUser, executionUser, date, invokeCode);
    newApplication.setProjectId(projectId);
    newApplication.setRuleGroupId(ruleGroupId);
    newApplication.setPartition(partition.toString());
    newApplication.setClusterName(clusterName);
    newApplication.setStartupParam(startupParam);
    newApplication.setExecutionParam(execParamStr);
    newApplication.setSetFlag(setFlag);
    ApplicationTaskSimpleResponse response;
    try {
        response = outerExecutionService.commonExecution(rules, partition, executionUser, nodeName, startupParam, clusterName, setFlag, execParams, newApplication, date, runDate);
    } catch (BothNullDatasourceException e) {
        catchAndSolve(e, ApplicationCommentEnum.BOTH_NULL_ISSUES.getCode(), ApplicationStatusEnum.FINISHED.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (LeftNullDatasourceException e) {
        catchAndSolve(e, ApplicationCommentEnum.LEFT_NULL_DATA_ISSUES.getCode(), ApplicationStatusEnum.NOT_PASS.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (RightNullDatasourceException e) {
        catchAndSolve(e, ApplicationCommentEnum.RIGHT_NULL_DATA_ISSUES.getCode(), ApplicationStatusEnum.NOT_PASS.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (MetaDataAcquireFailedException e) {
        catchAndSolve(e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", "{&THE_CHECK_FIELD_HAS_BEEN_MODIFIED}", null);
    } catch (DataSourceMoveException e) {
        catchAndSolve(e, ApplicationCommentEnum.PERMISSION_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (DataSourceOverSizeException e) {
        catchAndSolve(e, ApplicationCommentEnum.PERMISSION_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (ParseException e) {
        catchAndSolve(e, ApplicationCommentEnum.GRAMMAR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
        LOGGER.error(e.getMessage(), e);
        return new GeneralResponse<>("500", "{&PARSE_SQL_FAILED_PLEASE_CHECKOUT_YOUR_CUSTOM_SQL}", null);
    } catch (Exception e) {
        catchAndSolve(e, ApplicationCommentEnum.UNKNOWN_ERROR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    }
    LOGGER.info("Succeed to dispatch task. response: {}", response);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_DISPATCH_TASK}", response);
}