Search in sources :

Example 1 with ConvertException

use of com.webank.wedatasphere.qualitis.exception.ConvertException in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method submitApplication.

/**
 * Submit job to linkis
 */
@Override
public List<TaskSubmitResult> submitApplication(List<Rule> rules, String nodeName, String createTime, String user, String database, StringBuffer partition, Date date, Application application, String cluster, String startupParam, String setFlag, Map<String, String> execParams, StringBuffer runDate, Map<Long, Map> dataSourceMysqlConnect) throws ArgumentException, TaskTypeException, ConvertException, DataQualityTaskException, RuleVariableNotSupportException, RuleVariableNotFoundException, JobSubmitException, ClusterInfoNotConfigException, IOException, UnExpectedRequestException, MetaDataAcquireFailedException {
    String csId = rules.iterator().next().getCsId();
    // Check if cluster supported
    LOGGER.info("Start to collect rule to clusters");
    Map<String, List<Rule>> clusterNameMap = getRuleCluster(rules);
    LOGGER.info("Succeed to classify rules by cluster, cluster map: {}", clusterNameMap);
    if (StringUtils.isNotBlank(cluster)) {
        LOGGER.info("When pick up a cluster, these datasources of rules must be from one cluster. Now start to put into the specify cluster.\n");
        putAllRulesIntoSpecifyCluster(clusterNameMap, cluster);
        LOGGER.info("Success to put into the specify cluster.\n");
    }
    List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
    for (String clusterName : clusterNameMap.keySet()) {
        List<Rule> clusterRules = clusterNameMap.get(clusterName);
        if (StringUtils.isNotBlank(cluster)) {
            clusterName = cluster;
        }
        ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
        LOGGER.info("Start to check cluster config.");
        if (clusterInfo == null) {
            throw new ClusterInfoNotConfigException(clusterName + " {&DOES_NOT_EXIST}");
        }
        LOGGER.info("Succeed to pass the check of cluster config. All cluster of rules are configured");
        // Divide rule into tasks
        List<DataQualityTask> tasks = TaskDividerFactory.getDivider().divide(clusterRules, application.getId(), createTime, partition.toString(), date, database, user, taskExecuteLimitConfig.getTaskExecuteRuleSize());
        LOGGER.info("Succeed to divide application into tasks. result: {}", tasks);
        // Save divided tasks
        saveDividedTask(tasks, clusterInfo, rules, application, createTime);
        // Convert tasks into job
        List<DataQualityJob> jobList = new ArrayList<>();
        for (DataQualityTask task : tasks) {
            DataQualityJob job = templateConverterFactory.getConverter(task).convert(task, date, setFlag, execParams, runDate.toString(), clusterInfo.getClusterType(), dataSourceMysqlConnect);
            job.setUser(task.getUser());
            jobList.add(job);
            List<Long> ruleIdList = task.getRuleTaskDetails().stream().map(r -> r.getRule().getId()).collect(Collectors.toList());
            LOGGER.info("Succeed to convert rule_id: {} into code. code: {}", ruleIdList, job.getJobCode());
        }
        LOGGER.info("Succeed to convert all template into codes. codes: {}", jobList);
        // Submit job to linkis
        List<JobSubmitResult> submitResults = new ArrayList<>();
        for (DataQualityJob job : jobList) {
            String code = String.join("\n", job.getJobCode());
            String proxy = job.getUser();
            Long taskId = job.getTaskId();
            // Compatible with new and old submission interfaces.
            JobSubmitResult result = null;
            boolean engineReUse = false;
            if (StringUtils.isNotBlank(startupParam)) {
                String[] startupParams = startupParam.split(SpecCharEnum.DIVIDER.getValue());
                for (String param : startupParams) {
                    if (StringUtils.isEmpty(param)) {
                        continue;
                    }
                    String[] paramStrs = param.split("=");
                    if (paramStrs.length < 2) {
                        continue;
                    }
                    String key = paramStrs[0];
                    String value = paramStrs[1];
                    if ("engine_reuse".equals(key)) {
                        if ("true".equals(value)) {
                            engineReUse = true;
                            startupParam = startupParam.replace("engine_reuse=true", "");
                        } else {
                            engineReUse = false;
                            startupParam = startupParam.replace("engine_reuse=false", "");
                        }
                        break;
                    }
                }
            }
            if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
                result = abstractJobSubmitter.submitJobNew(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam(), engineReUse);
            } else {
                result = abstractJobSubmitter.submitJob(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam());
            }
            if (result != null) {
                submitResults.add(result);
            } else {
                Task taskInDb = taskDao.findById(taskId);
                taskInDb.setStatus(TaskStatusEnum.TASK_NOT_EXIST.getCode());
                taskDao.save(taskInDb);
                taskSubmitResults.add(new TaskSubmitResult(application.getId(), null, clusterInfo.getClusterName()));
            }
        }
        // Rewrite task remote ID.
        rewriteTaskRemoteInfo(submitResults, taskSubmitResults, application.getId(), clusterInfo.getClusterName());
    }
    return taskSubmitResults;
}
Also used : AlarmConfigStatusEnum(com.webank.wedatasphere.qualitis.constant.AlarmConfigStatusEnum) StringUtils(org.apache.commons.lang.StringUtils) ClusterInfoNotConfigException(com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException) Date(java.util.Date) TaskRuleDataSource(com.webank.wedatasphere.qualitis.bean.TaskRuleDataSource) LinkisConfig(com.webank.wedatasphere.qualitis.config.LinkisConfig) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) Application(com.webank.wedatasphere.qualitis.entity.Application) TemplateConverterFactory(com.webank.wedatasphere.qualitis.converter.TemplateConverterFactory) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) DateExprReplaceUtil(com.webank.wedatasphere.qualitis.util.DateExprReplaceUtil) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) ClusterInfoDao(com.webank.wedatasphere.qualitis.dao.ClusterInfoDao) AlarmConfig(com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig) Task(com.webank.wedatasphere.qualitis.entity.Task) TaskRuleAlarmConfig(com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig) Map(java.util.Map) LocaleParser(com.webank.wedatasphere.qualitis.parser.LocaleParser) JobSubmitException(com.webank.wedatasphere.qualitis.exception.JobSubmitException) ParseException(java.text.ParseException) FileOutputUnitEnum(com.webank.wedatasphere.qualitis.rule.constant.FileOutputUnitEnum) ConvertException(com.webank.wedatasphere.qualitis.exception.ConvertException) RuleVariableNotSupportException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotSupportException) RestClientException(org.springframework.web.client.RestClientException) ExecutionManager(com.webank.wedatasphere.qualitis.submitter.ExecutionManager) PartitionStatisticsInfo(com.webank.wedatasphere.qualitis.metadata.response.table.PartitionStatisticsInfo) Context(javax.ws.rs.core.Context) Set(java.util.Set) Collectors(java.util.stream.Collectors) MetaDataAcquireFailedException(com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException) DataQualityJob(com.webank.wedatasphere.qualitis.bean.DataQualityJob) List(java.util.List) TaskDividerFactory(com.webank.wedatasphere.qualitis.divider.TaskDividerFactory) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) TaskRule(com.webank.wedatasphere.qualitis.bean.TaskRule) MetaDataClient(com.webank.wedatasphere.qualitis.metadata.client.MetaDataClient) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) JobKillResult(com.webank.wedatasphere.qualitis.bean.JobKillResult) RuleTaskDetail(com.webank.wedatasphere.qualitis.bean.RuleTaskDetail) TaskRuleAlarmConfigBean(com.webank.wedatasphere.qualitis.bean.TaskRuleAlarmConfigBean) SpecCharEnum(com.webank.wedatasphere.qualitis.constant.SpecCharEnum) TaskStatusEnum(com.webank.wedatasphere.qualitis.constant.TaskStatusEnum) SimpleDateFormat(java.text.SimpleDateFormat) JobKillException(com.webank.wedatasphere.qualitis.exception.JobKillException) Rule(com.webank.wedatasphere.qualitis.rule.entity.Rule) HashMap(java.util.HashMap) JobSubmitResult(com.webank.wedatasphere.qualitis.bean.JobSubmitResult) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) HttpServletRequest(javax.servlet.http.HttpServletRequest) FilePassUtil(com.webank.wedatasphere.qualitis.util.FilePassUtil) TaskDao(com.webank.wedatasphere.qualitis.dao.TaskDao) TaskSubmitResult(com.webank.wedatasphere.qualitis.bean.TaskSubmitResult) DataQualityTaskException(com.webank.wedatasphere.qualitis.exception.DataQualityTaskException) TaskDataSourceRepository(com.webank.wedatasphere.qualitis.dao.repository.TaskDataSourceRepository) TaskTypeException(com.webank.wedatasphere.qualitis.exception.TaskTypeException) TaskResult(com.webank.wedatasphere.qualitis.entity.TaskResult) ArgumentException(com.webank.wedatasphere.qualitis.exception.ArgumentException) RuleVariableNotFoundException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotFoundException) AbstractJobSubmitter(com.webank.wedatasphere.qualitis.client.AbstractJobSubmitter) Logger(org.slf4j.Logger) RuleMetric(com.webank.wedatasphere.qualitis.entity.RuleMetric) TaskRuleSimpleRepository(com.webank.wedatasphere.qualitis.dao.repository.TaskRuleSimpleRepository) IOException(java.io.IOException) FastDateFormat(org.apache.commons.lang3.time.FastDateFormat) TableStatisticsInfo(com.webank.wedatasphere.qualitis.metadata.response.table.TableStatisticsInfo) FileOutputNameEnum(com.webank.wedatasphere.qualitis.rule.constant.FileOutputNameEnum) TaskRuleSimple(com.webank.wedatasphere.qualitis.entity.TaskRuleSimple) Component(org.springframework.stereotype.Component) ApplicationDao(com.webank.wedatasphere.qualitis.dao.ApplicationDao) DataQualityTask(com.webank.wedatasphere.qualitis.bean.DataQualityTask) UnitTransfer(com.webank.wedatasphere.qualitis.rule.util.UnitTransfer) TaskExecuteLimitConfig(com.webank.wedatasphere.qualitis.config.TaskExecuteLimitConfig) TaskResultDao(com.webank.wedatasphere.qualitis.dao.TaskResultDao) TaskDataSource(com.webank.wedatasphere.qualitis.entity.TaskDataSource) Task(com.webank.wedatasphere.qualitis.entity.Task) DataQualityTask(com.webank.wedatasphere.qualitis.bean.DataQualityTask) ArrayList(java.util.ArrayList) JobSubmitResult(com.webank.wedatasphere.qualitis.bean.JobSubmitResult) ClusterInfoNotConfigException(com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException) DataQualityTask(com.webank.wedatasphere.qualitis.bean.DataQualityTask) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) List(java.util.List) ArrayList(java.util.ArrayList) TaskRule(com.webank.wedatasphere.qualitis.bean.TaskRule) Rule(com.webank.wedatasphere.qualitis.rule.entity.Rule) DataQualityJob(com.webank.wedatasphere.qualitis.bean.DataQualityJob) TaskSubmitResult(com.webank.wedatasphere.qualitis.bean.TaskSubmitResult)

Example 2 with ConvertException

use of com.webank.wedatasphere.qualitis.exception.ConvertException in project Qualitis by WeBankFinTech.

the class SqlTemplateConverter method generateSparkSqlByTask.

/**
 * Convert task into scala code
 * @param rule
 * @param date
 * @param applicationId
 * @param midTableName
 * @param createTime
 * @param partition
 * @param execParams
 * @param count
 * @param runDate
 * @param dataSourceMysqlConnect
 * @return
 * @throws ConvertException
 * @throws RuleVariableNotSupportException
 * @throws RuleVariableNotFoundException
 */
private List<String> generateSparkSqlByTask(Rule rule, Date date, String applicationId, String midTableName, String createTime, StringBuffer partition, Map<String, String> execParams, int count, String runDate, Map<Long, Map> dataSourceMysqlConnect) throws ConvertException, RuleVariableNotSupportException, RuleVariableNotFoundException, UnExpectedRequestException {
    List<String> sqlList = new ArrayList<>();
    // Collect rule metric and build in save sentence sql.
    List<RuleMetric> ruleMetrics = rule.getAlarmConfigs().stream().map(AlarmConfig::getRuleMetric).distinct().collect(Collectors.toList());
    Map<String, Long> ruleMetricMap = new HashMap<>(ruleMetrics.size());
    if (CollectionUtils.isNotEmpty(ruleMetrics)) {
        LOGGER.info("Start to get rule metric for task result save. Rule metrics: {}", Arrays.toString(ruleMetrics.toArray()));
        for (RuleMetric ruleMetric : ruleMetrics) {
            if (ruleMetric != null) {
                ruleMetricMap.put(ruleMetric.getName(), ruleMetric.getId());
            }
        }
        LOGGER.info("Finish to get rule metric for task result save.");
    }
    // Get SQL from template after remove '\n'
    String templateMidTableAction = rule.getTemplate().getMidTableAction().replace("\n", " ");
    Map<String, String> filters = new HashMap<>(2);
    if (CUSTOM_RULE.intValue() == rule.getRuleType()) {
        templateMidTableAction = customMidTableActionUpdate(rule, templateMidTableAction, date, execParams, partition, ruleMetricMap);
    } else if (MUL_SOURCE_RULE.intValue() == rule.getRuleType()) {
        templateMidTableAction = multiMidTableActionUpdate(rule, templateMidTableAction, date, filters);
    }
    // Get input meta from template
    List<RuleVariable> inputMetaRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.TEMPLATE_INPUT_META.getCode())).collect(Collectors.toList());
    // If partition is not specified, replace with filter in rule configuration.
    if (StringUtils.isBlank(partition.toString())) {
        templateMidTableAction = fillPartitionWithRuleConfiguration(partition, rule, templateMidTableAction, inputMetaRuleVariables);
    }
    // Get dbs and tables
    Map<String, String> dbTableMap = new HashMap<>(4);
    // Get mappings
    StringBuffer mappings = new StringBuffer();
    StringBuffer realFilter = new StringBuffer();
    // Get SQL From template and replace all replaceholders
    String midTableAction = replaceVariable(templateMidTableAction, inputMetaRuleVariables, partition.toString(), realFilter, dbTableMap, mappings, date);
    Set<TemplateStatisticsInputMeta> templateStatisticsAction = rule.getTemplate().getStatisticAction();
    Map sourceConnect = new HashMap(8);
    Map targetConnect = new HashMap(8);
    if (dataSourceMysqlConnect != null && dataSourceMysqlConnect.size() > 0) {
        for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
            Map connectParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
            if (connectParams == null) {
                continue;
            }
            if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(0)) {
                // If mysql sec, decrypt password and user name.
                sourceConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
            }
            if (ruleDataSource.getDatasourceIndex() != null && ruleDataSource.getDatasourceIndex().equals(1)) {
                // If mysql sec, decrypt password and user name.
                targetConnect = dataSourceMysqlConnect.get(ruleDataSource.getId());
            }
        }
    }
    sqlList.add("val UUID = java.util.UUID.randomUUID.toString");
    // 跨表规则
    if (RuleTemplateTypeEnum.MULTI_SOURCE_TEMPLATE.getCode().equals(rule.getTemplate().getTemplateType()) && dbTableMap.size() > 0) {
        // Import sql function.
        sqlList.addAll(getImportSql());
        // Generate UUID.
        // Transform original table.
        Set<String> columns = new HashSet<>();
        if (rule.getTemplate().getId().longValue() == MUL_SOURCE_ACCURACY_TEMPLATE_ID.longValue()) {
            // Get accuracy columns.
            columns = rule.getRuleDataSourceMappings().stream().map(RuleDataSourceMapping::getLeftColumnNames).map(column -> column.replace("tmp1.", "").replace("tmp2.", "")).collect(Collectors.toSet());
        }
        if (rule.getTemplate().getId().longValue() == MUL_SOURCE_COMMON_TEMPLATE_ID.longValue()) {
            sqlList.addAll(getCommonTransformSql(dbTableMap, mappings, count, partition.toString(), filters, sourceConnect, targetConnect));
        } else {
            sqlList.addAll(getSpecialTransformSql(dbTableMap, count, partition.toString(), filters, Strings.join(columns, ','), sourceConnect, targetConnect));
            if (optimizationConfig.getLightweightQuery()) {
                count += 3;
            }
        }
        sqlList.addAll(getSaveMidTableSentenceSettings());
        sqlList.addAll(getSaveMidTableSentence(midTableName, count, runDate));
    } else {
        // Generate select statement and save into hive database
        RuleDataSource ruleDataSource = rule.getRuleDataSources().stream().filter(dataSource -> dataSource.getDatasourceIndex() == null).iterator().next();
        Map connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
        if (connParams != null) {
            connParams = dataSourceMysqlConnect.get(ruleDataSource.getId());
        }
        sqlList.addAll(generateSparkSqlAndSaveSentence(midTableAction, midTableName, rule.getTemplate(), count, connParams, runDate));
        count++;
    }
    // Generate statistics statement, and save into mysql
    List<RuleVariable> statisticsRuleVariables = rule.getRuleVariables().stream().filter(ruleVariable -> ruleVariable.getInputActionStep().equals(InputActionStepEnum.STATISTICS_ARG.getCode())).collect(Collectors.toList());
    sqlList.addAll(saveStatisticAndSaveMySqlSentence(rule.getId(), ruleMetricMap, templateStatisticsAction, applicationId, statisticsRuleVariables, createTime, count, runDate));
    return sqlList;
}
Also used : RuleMetric(com.webank.wedatasphere.qualitis.entity.RuleMetric) Arrays(java.util.Arrays) RuleTaskDetail(com.webank.wedatasphere.qualitis.bean.RuleTaskDetail) SpecCharEnum(com.webank.wedatasphere.qualitis.constant.SpecCharEnum) Date(java.util.Date) RuleVariable(com.webank.wedatasphere.qualitis.rule.entity.RuleVariable) LoggerFactory(org.slf4j.LoggerFactory) SimpleDateFormat(java.text.SimpleDateFormat) Autowired(org.springframework.beans.factory.annotation.Autowired) Rule(com.webank.wedatasphere.qualitis.rule.entity.Rule) HashMap(java.util.HashMap) StringUtils(org.apache.commons.lang3.StringUtils) ArrayList(java.util.ArrayList) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) DateExprReplaceUtil(com.webank.wedatasphere.qualitis.util.DateExprReplaceUtil) HashSet(java.util.HashSet) AlarmConfig(com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig) Matcher(java.util.regex.Matcher) CollectionUtils(org.apache.commons.collections.CollectionUtils) DataQualityTaskException(com.webank.wedatasphere.qualitis.exception.DataQualityTaskException) Map(java.util.Map) ConvertException(com.webank.wedatasphere.qualitis.exception.ConvertException) RuleVariableNotSupportException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotSupportException) RuleVariableNotFoundException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotFoundException) TemplateMidTableInputMeta(com.webank.wedatasphere.qualitis.rule.entity.TemplateMidTableInputMeta) Logger(org.slf4j.Logger) InputActionStepEnum(com.webank.wedatasphere.qualitis.rule.constant.InputActionStepEnum) AbstractTranslator(com.webank.wedatasphere.qualitis.translator.AbstractTranslator) RuleMetric(com.webank.wedatasphere.qualitis.entity.RuleMetric) Template(com.webank.wedatasphere.qualitis.rule.entity.Template) Set(java.util.Set) IOException(java.io.IOException) Collectors(java.util.stream.Collectors) RuleDataSourceMapping(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSourceMapping) RuleConstraintEnum(com.webank.wedatasphere.qualitis.metadata.constant.RuleConstraintEnum) DataQualityJob(com.webank.wedatasphere.qualitis.bean.DataQualityJob) TemplateInputTypeEnum(com.webank.wedatasphere.qualitis.rule.constant.TemplateInputTypeEnum) List(java.util.List) Component(org.springframework.stereotype.Component) RuleTemplateTypeEnum(com.webank.wedatasphere.qualitis.rule.constant.RuleTemplateTypeEnum) Strings(org.apache.logging.log4j.util.Strings) DataQualityTask(com.webank.wedatasphere.qualitis.bean.DataQualityTask) TemplateStatisticsInputMeta(com.webank.wedatasphere.qualitis.rule.entity.TemplateStatisticsInputMeta) Pattern(java.util.regex.Pattern) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) OptimizationConfig(com.webank.wedatasphere.qualitis.config.OptimizationConfig) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) TemplateStatisticsInputMeta(com.webank.wedatasphere.qualitis.rule.entity.TemplateStatisticsInputMeta) HashMap(java.util.HashMap) Map(java.util.Map) AlarmConfig(com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig) RuleVariable(com.webank.wedatasphere.qualitis.rule.entity.RuleVariable) HashSet(java.util.HashSet)

Example 3 with ConvertException

use of com.webank.wedatasphere.qualitis.exception.ConvertException in project Qualitis by WeBankFinTech.

the class SqlTemplateConverter method replaceVariable.

/**
 * Replace all placeholder of template sql
 * @param template
 * @param variables
 * @param filter
 * @param realFilter
 * @param dbTableMap for pick up source db.table & target db.table
 * @param mappings
 * @param date
 * @return
 * @throws ConvertException
 */
private String replaceVariable(String template, List<RuleVariable> variables, String filter, StringBuffer realFilter, Map<String, String> dbTableMap, StringBuffer mappings, Date date) throws ConvertException, UnExpectedRequestException {
    String sqlAction = template;
    if (StringUtils.isNotBlank(filter)) {
        String tmpfilter = DateExprReplaceUtil.replaceFilter(date, filter);
        sqlAction = sqlAction.replace(FILTER_PLACEHOLDER, tmpfilter);
        realFilter.append(tmpfilter);
        LOGGER.info("Succeed to replace {} into {}", FILTER_PLACEHOLDER, tmpfilter);
    } else {
        realFilter.append("true");
    }
    for (RuleVariable ruleVariable : variables) {
        String midInputMetaPlaceHolder = ruleVariable.getTemplateMidTableInputMeta().getPlaceholder();
        String placeHolder = "\\$\\{" + midInputMetaPlaceHolder + "}";
        // GeT source db and table, target db and table.
        if ("source_db".equals(midInputMetaPlaceHolder)) {
            if (StringUtils.isNotBlank(ruleVariable.getValue())) {
                dbTableMap.put("source_db", ruleVariable.getValue() + ".");
            } else {
                dbTableMap.put("source_db", "");
            }
        } else if ("source_table".equals(midInputMetaPlaceHolder)) {
            dbTableMap.put("source_table", ruleVariable.getValue());
        } else if ("target_table".equals(midInputMetaPlaceHolder)) {
            dbTableMap.put("target_table", ruleVariable.getValue());
        } else if ("target_db".equals(midInputMetaPlaceHolder)) {
            if (StringUtils.isNotBlank(ruleVariable.getValue())) {
                dbTableMap.put("target_db", ruleVariable.getValue() + ".");
            } else {
                dbTableMap.put("target_db", "");
            }
        } else if ("mapping_argument".equals(midInputMetaPlaceHolder)) {
            mappings.append(ruleVariable.getValue());
        }
        // Fix issue of wedget node in the front.
        if ("\\$\\{field}".equals(placeHolder)) {
            Matcher matcher = AGGREGATE_FUNC_PATTERN.matcher(ruleVariable.getValue());
            while (matcher.find()) {
                String[] funcs = matcher.group().split("\n");
                for (String func : funcs) {
                    ruleVariable.setValue(ruleVariable.getValue().replace(func, "`" + func + "`"));
                }
            }
        }
        // Fix replacement issue that db is null when running workflow.
        if (ruleVariable.getValue() == null || "".equals(ruleVariable.getValue())) {
            sqlAction = sqlAction.replaceAll(placeHolder + ".", "");
        } else {
            sqlAction = sqlAction.replaceAll(placeHolder, ruleVariable.getValue());
        }
        LOGGER.info("Succeed to replace {} into {}", placeHolder, ruleVariable.getValue());
    }
    if (PLACEHOLDER_PATTERN.matcher(sqlAction).matches()) {
        throw new ConvertException("Unable to convert SQL, replacing placeholders failed, still having placeholder.");
    }
    return sqlAction;
}
Also used : ConvertException(com.webank.wedatasphere.qualitis.exception.ConvertException) Matcher(java.util.regex.Matcher) RuleVariable(com.webank.wedatasphere.qualitis.rule.entity.RuleVariable)

Aggregations

ConvertException (com.webank.wedatasphere.qualitis.exception.ConvertException)3 DataQualityJob (com.webank.wedatasphere.qualitis.bean.DataQualityJob)2 DataQualityTask (com.webank.wedatasphere.qualitis.bean.DataQualityTask)2 RuleTaskDetail (com.webank.wedatasphere.qualitis.bean.RuleTaskDetail)2 SpecCharEnum (com.webank.wedatasphere.qualitis.constant.SpecCharEnum)2 RuleMetric (com.webank.wedatasphere.qualitis.entity.RuleMetric)2 DataQualityTaskException (com.webank.wedatasphere.qualitis.exception.DataQualityTaskException)2 RuleVariableNotFoundException (com.webank.wedatasphere.qualitis.exception.RuleVariableNotFoundException)2 RuleVariableNotSupportException (com.webank.wedatasphere.qualitis.exception.RuleVariableNotSupportException)2 RuleVariable (com.webank.wedatasphere.qualitis.rule.entity.RuleVariable)2 Matcher (java.util.regex.Matcher)2 JobKillResult (com.webank.wedatasphere.qualitis.bean.JobKillResult)1 JobSubmitResult (com.webank.wedatasphere.qualitis.bean.JobSubmitResult)1 TaskRule (com.webank.wedatasphere.qualitis.bean.TaskRule)1 TaskRuleAlarmConfigBean (com.webank.wedatasphere.qualitis.bean.TaskRuleAlarmConfigBean)1 TaskRuleDataSource (com.webank.wedatasphere.qualitis.bean.TaskRuleDataSource)1 TaskSubmitResult (com.webank.wedatasphere.qualitis.bean.TaskSubmitResult)1 AbstractJobSubmitter (com.webank.wedatasphere.qualitis.client.AbstractJobSubmitter)1 LinkisConfig (com.webank.wedatasphere.qualitis.config.LinkisConfig)1 OptimizationConfig (com.webank.wedatasphere.qualitis.config.OptimizationConfig)1