Examples with ArgumentException com.webank.wedatasphere.qualitis.exception.ArgumentException used on opensource projects

Example 1 with ArgumentException

use of com.webank.wedatasphere.qualitis.exception.ArgumentException in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method submitApplication.

/**
 * Submit job to linkis
 */
@Override
public List<TaskSubmitResult> submitApplication(List<Rule> rules, String nodeName, String createTime, String user, String database, StringBuffer partition, Date date, Application application, String cluster, String startupParam, String setFlag, Map<String, String> execParams, StringBuffer runDate, Map<Long, Map> dataSourceMysqlConnect) throws ArgumentException, TaskTypeException, ConvertException, DataQualityTaskException, RuleVariableNotSupportException, RuleVariableNotFoundException, JobSubmitException, ClusterInfoNotConfigException, IOException, UnExpectedRequestException, MetaDataAcquireFailedException {
    String csId = rules.iterator().next().getCsId();
    // Check if cluster supported
    LOGGER.info("Start to collect rule to clusters");
    Map<String, List<Rule>> clusterNameMap = getRuleCluster(rules);
    LOGGER.info("Succeed to classify rules by cluster, cluster map: {}", clusterNameMap);
    if (StringUtils.isNotBlank(cluster)) {
        LOGGER.info("When pick up a cluster, these datasources of rules must be from one cluster. Now start to put into the specify cluster.\n");
        putAllRulesIntoSpecifyCluster(clusterNameMap, cluster);
        LOGGER.info("Success to put into the specify cluster.\n");
    }
    List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
    for (String clusterName : clusterNameMap.keySet()) {
        List<Rule> clusterRules = clusterNameMap.get(clusterName);
        if (StringUtils.isNotBlank(cluster)) {
            clusterName = cluster;
        }
        ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
        LOGGER.info("Start to check cluster config.");
        if (clusterInfo == null) {
            throw new ClusterInfoNotConfigException(clusterName + " {&DOES_NOT_EXIST}");
        }
        LOGGER.info("Succeed to pass the check of cluster config. All cluster of rules are configured");
        // Divide rule into tasks
        List<DataQualityTask> tasks = TaskDividerFactory.getDivider().divide(clusterRules, application.getId(), createTime, partition.toString(), date, database, user, taskExecuteLimitConfig.getTaskExecuteRuleSize());
        LOGGER.info("Succeed to divide application into tasks. result: {}", tasks);
        // Save divided tasks
        saveDividedTask(tasks, clusterInfo, rules, application, createTime);
        // Convert tasks into job
        List<DataQualityJob> jobList = new ArrayList<>();
        for (DataQualityTask task : tasks) {
            DataQualityJob job = templateConverterFactory.getConverter(task).convert(task, date, setFlag, execParams, runDate.toString(), clusterInfo.getClusterType(), dataSourceMysqlConnect);
            job.setUser(task.getUser());
            jobList.add(job);
            List<Long> ruleIdList = task.getRuleTaskDetails().stream().map(r -> r.getRule().getId()).collect(Collectors.toList());
            LOGGER.info("Succeed to convert rule_id: {} into code. code: {}", ruleIdList, job.getJobCode());
        }
        LOGGER.info("Succeed to convert all template into codes. codes: {}", jobList);
        // Submit job to linkis
        List<JobSubmitResult> submitResults = new ArrayList<>();
        for (DataQualityJob job : jobList) {
            String code = String.join("\n", job.getJobCode());
            String proxy = job.getUser();
            Long taskId = job.getTaskId();
            // Compatible with new and old submission interfaces.
            JobSubmitResult result = null;
            boolean engineReUse = false;
            if (StringUtils.isNotBlank(startupParam)) {
                String[] startupParams = startupParam.split(SpecCharEnum.DIVIDER.getValue());
                for (String param : startupParams) {
                    if (StringUtils.isEmpty(param)) {
                        continue;
                    }
                    String[] paramStrs = param.split("=");
                    if (paramStrs.length < 2) {
                        continue;
                    }
                    String key = paramStrs[0];
                    String value = paramStrs[1];
                    if ("engine_reuse".equals(key)) {
                        if ("true".equals(value)) {
                            engineReUse = true;
                            startupParam = startupParam.replace("engine_reuse=true", "");
                        } else {
                            engineReUse = false;
                            startupParam = startupParam.replace("engine_reuse=false", "");
                        }
                        break;
                    }
                }
            }
            if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
                result = abstractJobSubmitter.submitJobNew(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam(), engineReUse);
            } else {
                result = abstractJobSubmitter.submitJob(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam());
            }
            if (result != null) {
                submitResults.add(result);
            } else {
                Task taskInDb = taskDao.findById(taskId);
                taskInDb.setStatus(TaskStatusEnum.TASK_NOT_EXIST.getCode());
                taskDao.save(taskInDb);
                taskSubmitResults.add(new TaskSubmitResult(application.getId(), null, clusterInfo.getClusterName()));
            }
        }
        // Rewrite task remote ID.
        rewriteTaskRemoteInfo(submitResults, taskSubmitResults, application.getId(), clusterInfo.getClusterName());
    }
    return taskSubmitResults;
}

Example 2 with ArgumentException

use of com.webank.wedatasphere.qualitis.exception.ArgumentException in project Qualitis by WeBankFinTech.

the class SameDataSourceTaskDivider method divide.

@Override
public List<DataQualityTask> divide(List<Rule> rules, String applicationId, String createTime, String partition, Date date, String database, String user, Integer threshold) throws ArgumentException {
    LOGGER.info("Start to classify rules by datasource");
    Map<String, List<Rule>> sameDataSourceRule = new HashMap<>(4);
    for (Rule rule : rules) {
        String key = getKey(rule, user);
        // Rules without specific execution parameters can be split into the same task, and rules with execution parameters must be treated as a separate task.
        Boolean specifyStaticStartupParam = (rule.getSpecifyStaticStartupParam() != null && rule.getSpecifyStaticStartupParam());
        if (sameDataSourceRule.containsKey(key) && !specifyStaticStartupParam) {
            sameDataSourceRule.get(key).add(rule);
        } else if (specifyStaticStartupParam) {
            List<Rule> tmp = new ArrayList<>();
            tmp.add(rule);
            sameDataSourceRule.put(UUID.randomUUID().toString().replace("-", "") + "." + key, tmp);
        } else {
            List<Rule> tmp = new ArrayList<>();
            tmp.add(rule);
            sameDataSourceRule.put(key, tmp);
        }
    }
    LOGGER.info("Succeed to classify rules by datasource. Result: {}", sameDataSourceRule);
    List<DataQualityTask> result = new ArrayList<>();
    for (String key : sameDataSourceRule.keySet()) {
        List<Rule> ruleList = sameDataSourceRule.get(key);
        String ruleStartup = ruleList.stream().map(Rule::getStaticStartupParam).filter(staticStartupParam -> StringUtils.isNotBlank(staticStartupParam)).collect(Collectors.joining());
        List<Long> ruleIdList = ruleList.stream().map(Rule::getId).collect(Collectors.toList());
        LOGGER.info("Start to divide rules: {} into a task.", ruleIdList);
        LOGGER.info("Start to divide rules. Key: {}", key);
        String[] keys = key.split("\\.");
        String proxyUser = keys[keys.length - 1];
        List<RuleTaskDetail> ruleTaskDetails = new ArrayList<>();
        if (StringUtils.isNotBlank(proxyUser) && database.contains("_ind")) {
            database = proxyUser.concat("_ind");
        }
        for (Rule rule : ruleList) {
            String tableName = generateTable(rule);
            String midTableName = database + "." + tableName;
            if (ruleTaskDetails.size() < threshold) {
                ruleTaskDetails.add(new RuleTaskDetail(rule, midTableName));
            } else {
                List<RuleTaskDetail> ruleTaskDetailCopy = new ArrayList<>();
                ruleTaskDetailCopy.addAll(ruleTaskDetails);
                DataQualityTask tmp = new DataQualityTask(applicationId, createTime, partition, ruleTaskDetailCopy);
                if (StringUtils.isNotBlank(ruleStartup)) {
                    tmp.setStartupParam(ruleStartup);
                }
                if (StringUtils.isNotBlank(proxyUser)) {
                    LOGGER.info("Start to divide rules. Proxy user: {}", proxyUser);
                    tmp.setUser(proxyUser);
                }
                result.add(tmp);
                ruleTaskDetails = new ArrayList<>();
            }
        }
        if (ruleTaskDetails.size() > 0) {
            DataQualityTask tmp = new DataQualityTask(applicationId, createTime, partition, ruleTaskDetails);
            if (StringUtils.isNotBlank(ruleStartup)) {
                tmp.setStartupParam(ruleStartup);
            }
            if (StringUtils.isNotBlank(proxyUser)) {
                tmp.setUser(proxyUser);
            }
            result.add(tmp);
            LOGGER.info("Succeed to divide rules: {} into a task {}", ruleIdList, tmp);
        }
    }
    LOGGER.info("Succeed to divide all rules into tasks. result: {}", result);
    return result;
}

Example 3 with ArgumentException

use of com.webank.wedatasphere.qualitis.exception.ArgumentException in project Qualitis by WeBankFinTech.

the class SameDataSourceTaskDivider method getKey.

private String getKey(Rule rule, String user) throws ArgumentException {
    if (rule.getRuleDataSources().size() != 0) {
        List<RuleDataSource> ruleDataSourceList = rule.getRuleDataSources().stream().filter(dataSource -> StringUtils.isNotBlank(dataSource.getDbName())).collect(Collectors.toList());
        RuleDataSource ruleDataSource;
        if (CollectionUtils.isNotEmpty(ruleDataSourceList)) {
            ruleDataSource = ruleDataSourceList.iterator().next();
        } else {
            ruleDataSource = rule.getRuleDataSources().iterator().next();
        }
        String proxyUser = ruleDataSource.getProxyUser();
        if (StringUtils.isNotBlank(proxyUser)) {
            return ruleDataSource.getClusterName() + "." + ruleDataSource.getDbName() + "." + proxyUser;
        }
        return ruleDataSource.getClusterName() + "." + ruleDataSource.getDbName() + "." + user;
    }
    throw new ArgumentException("Error! Rule variables miss data");
}