Search in sources :

Example 1 with ClusterInfo

use of com.webank.wedatasphere.qualitis.entity.ClusterInfo in project Qualitis by WeBankFinTech.

the class RuleDataSourceServiceImpl method checkDataSourceClusterSupport.

/**
 * Check if cluster name supported
 * return if there is no cluster config
 * @param submittedClusterNames
 * @throws UnExpectedRequestException
 */
@Override
public void checkDataSourceClusterSupport(Set<String> submittedClusterNames) throws UnExpectedRequestException {
    if (submittedClusterNames == null || submittedClusterNames.isEmpty()) {
        return;
    }
    List<ClusterInfo> clusters = clusterInfoDao.findAllClusterInfo(0, Integer.MAX_VALUE);
    if (clusters == null || clusters.isEmpty()) {
        LOGGER.info("Failed to find cluster info config. End to check the limitation of cluster info.");
        return;
    }
    Set<String> supportClusterNames = new HashSet<>();
    for (ClusterInfo info : clusters) {
        supportClusterNames.add(info.getClusterName());
    }
    Set<String> unSupportClusterNameSet = new HashSet<>();
    for (String clusterName : submittedClusterNames) {
        if (!supportClusterNames.contains(clusterName)) {
            unSupportClusterNameSet.add(clusterName);
        }
    }
    if (unSupportClusterNameSet.size() > 0) {
        throw new UnExpectedRequestException(String.format("{&NOT_SUPPORT_CLUSTER_NAME}:%s,{&ONLY_SUPPORT_CLUSTER_NAME_ARE}:%s", unSupportClusterNameSet, submittedClusterNames.toString()));
    }
}
Also used : UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) HashSet(java.util.HashSet)

Example 2 with ClusterInfo

use of com.webank.wedatasphere.qualitis.entity.ClusterInfo in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method submitApplication.

/**
 * Submit job to linkis
 */
@Override
public List<TaskSubmitResult> submitApplication(List<Rule> rules, String nodeName, String createTime, String user, String database, StringBuffer partition, Date date, Application application, String cluster, String startupParam, String setFlag, Map<String, String> execParams, StringBuffer runDate, Map<Long, Map> dataSourceMysqlConnect) throws ArgumentException, TaskTypeException, ConvertException, DataQualityTaskException, RuleVariableNotSupportException, RuleVariableNotFoundException, JobSubmitException, ClusterInfoNotConfigException, IOException, UnExpectedRequestException, MetaDataAcquireFailedException {
    String csId = rules.iterator().next().getCsId();
    // Check if cluster supported
    LOGGER.info("Start to collect rule to clusters");
    Map<String, List<Rule>> clusterNameMap = getRuleCluster(rules);
    LOGGER.info("Succeed to classify rules by cluster, cluster map: {}", clusterNameMap);
    if (StringUtils.isNotBlank(cluster)) {
        LOGGER.info("When pick up a cluster, these datasources of rules must be from one cluster. Now start to put into the specify cluster.\n");
        putAllRulesIntoSpecifyCluster(clusterNameMap, cluster);
        LOGGER.info("Success to put into the specify cluster.\n");
    }
    List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
    for (String clusterName : clusterNameMap.keySet()) {
        List<Rule> clusterRules = clusterNameMap.get(clusterName);
        if (StringUtils.isNotBlank(cluster)) {
            clusterName = cluster;
        }
        ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
        LOGGER.info("Start to check cluster config.");
        if (clusterInfo == null) {
            throw new ClusterInfoNotConfigException(clusterName + " {&DOES_NOT_EXIST}");
        }
        LOGGER.info("Succeed to pass the check of cluster config. All cluster of rules are configured");
        // Divide rule into tasks
        List<DataQualityTask> tasks = TaskDividerFactory.getDivider().divide(clusterRules, application.getId(), createTime, partition.toString(), date, database, user, taskExecuteLimitConfig.getTaskExecuteRuleSize());
        LOGGER.info("Succeed to divide application into tasks. result: {}", tasks);
        // Save divided tasks
        saveDividedTask(tasks, clusterInfo, rules, application, createTime);
        // Convert tasks into job
        List<DataQualityJob> jobList = new ArrayList<>();
        for (DataQualityTask task : tasks) {
            DataQualityJob job = templateConverterFactory.getConverter(task).convert(task, date, setFlag, execParams, runDate.toString(), clusterInfo.getClusterType(), dataSourceMysqlConnect);
            job.setUser(task.getUser());
            jobList.add(job);
            List<Long> ruleIdList = task.getRuleTaskDetails().stream().map(r -> r.getRule().getId()).collect(Collectors.toList());
            LOGGER.info("Succeed to convert rule_id: {} into code. code: {}", ruleIdList, job.getJobCode());
        }
        LOGGER.info("Succeed to convert all template into codes. codes: {}", jobList);
        // Submit job to linkis
        List<JobSubmitResult> submitResults = new ArrayList<>();
        for (DataQualityJob job : jobList) {
            String code = String.join("\n", job.getJobCode());
            String proxy = job.getUser();
            Long taskId = job.getTaskId();
            // Compatible with new and old submission interfaces.
            JobSubmitResult result = null;
            boolean engineReUse = false;
            if (StringUtils.isNotBlank(startupParam)) {
                String[] startupParams = startupParam.split(SpecCharEnum.DIVIDER.getValue());
                for (String param : startupParams) {
                    if (StringUtils.isEmpty(param)) {
                        continue;
                    }
                    String[] paramStrs = param.split("=");
                    if (paramStrs.length < 2) {
                        continue;
                    }
                    String key = paramStrs[0];
                    String value = paramStrs[1];
                    if ("engine_reuse".equals(key)) {
                        if ("true".equals(value)) {
                            engineReUse = true;
                            startupParam = startupParam.replace("engine_reuse=true", "");
                        } else {
                            engineReUse = false;
                            startupParam = startupParam.replace("engine_reuse=false", "");
                        }
                        break;
                    }
                }
            }
            if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
                result = abstractJobSubmitter.submitJobNew(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam(), engineReUse);
            } else {
                result = abstractJobSubmitter.submitJob(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam());
            }
            if (result != null) {
                submitResults.add(result);
            } else {
                Task taskInDb = taskDao.findById(taskId);
                taskInDb.setStatus(TaskStatusEnum.TASK_NOT_EXIST.getCode());
                taskDao.save(taskInDb);
                taskSubmitResults.add(new TaskSubmitResult(application.getId(), null, clusterInfo.getClusterName()));
            }
        }
        // Rewrite task remote ID.
        rewriteTaskRemoteInfo(submitResults, taskSubmitResults, application.getId(), clusterInfo.getClusterName());
    }
    return taskSubmitResults;
}
Also used : AlarmConfigStatusEnum(com.webank.wedatasphere.qualitis.constant.AlarmConfigStatusEnum) StringUtils(org.apache.commons.lang.StringUtils) ClusterInfoNotConfigException(com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException) Date(java.util.Date) TaskRuleDataSource(com.webank.wedatasphere.qualitis.bean.TaskRuleDataSource) LinkisConfig(com.webank.wedatasphere.qualitis.config.LinkisConfig) LoggerFactory(org.slf4j.LoggerFactory) Autowired(org.springframework.beans.factory.annotation.Autowired) Application(com.webank.wedatasphere.qualitis.entity.Application) TemplateConverterFactory(com.webank.wedatasphere.qualitis.converter.TemplateConverterFactory) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) DateExprReplaceUtil(com.webank.wedatasphere.qualitis.util.DateExprReplaceUtil) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) ClusterInfoDao(com.webank.wedatasphere.qualitis.dao.ClusterInfoDao) AlarmConfig(com.webank.wedatasphere.qualitis.rule.entity.AlarmConfig) Task(com.webank.wedatasphere.qualitis.entity.Task) TaskRuleAlarmConfig(com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig) Map(java.util.Map) LocaleParser(com.webank.wedatasphere.qualitis.parser.LocaleParser) JobSubmitException(com.webank.wedatasphere.qualitis.exception.JobSubmitException) ParseException(java.text.ParseException) FileOutputUnitEnum(com.webank.wedatasphere.qualitis.rule.constant.FileOutputUnitEnum) ConvertException(com.webank.wedatasphere.qualitis.exception.ConvertException) RuleVariableNotSupportException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotSupportException) RestClientException(org.springframework.web.client.RestClientException) ExecutionManager(com.webank.wedatasphere.qualitis.submitter.ExecutionManager) PartitionStatisticsInfo(com.webank.wedatasphere.qualitis.metadata.response.table.PartitionStatisticsInfo) Context(javax.ws.rs.core.Context) Set(java.util.Set) Collectors(java.util.stream.Collectors) MetaDataAcquireFailedException(com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException) DataQualityJob(com.webank.wedatasphere.qualitis.bean.DataQualityJob) List(java.util.List) TaskDividerFactory(com.webank.wedatasphere.qualitis.divider.TaskDividerFactory) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) TaskRule(com.webank.wedatasphere.qualitis.bean.TaskRule) MetaDataClient(com.webank.wedatasphere.qualitis.metadata.client.MetaDataClient) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) JobKillResult(com.webank.wedatasphere.qualitis.bean.JobKillResult) RuleTaskDetail(com.webank.wedatasphere.qualitis.bean.RuleTaskDetail) TaskRuleAlarmConfigBean(com.webank.wedatasphere.qualitis.bean.TaskRuleAlarmConfigBean) SpecCharEnum(com.webank.wedatasphere.qualitis.constant.SpecCharEnum) TaskStatusEnum(com.webank.wedatasphere.qualitis.constant.TaskStatusEnum) SimpleDateFormat(java.text.SimpleDateFormat) JobKillException(com.webank.wedatasphere.qualitis.exception.JobKillException) Rule(com.webank.wedatasphere.qualitis.rule.entity.Rule) HashMap(java.util.HashMap) JobSubmitResult(com.webank.wedatasphere.qualitis.bean.JobSubmitResult) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) HttpServletRequest(javax.servlet.http.HttpServletRequest) FilePassUtil(com.webank.wedatasphere.qualitis.util.FilePassUtil) TaskDao(com.webank.wedatasphere.qualitis.dao.TaskDao) TaskSubmitResult(com.webank.wedatasphere.qualitis.bean.TaskSubmitResult) DataQualityTaskException(com.webank.wedatasphere.qualitis.exception.DataQualityTaskException) TaskDataSourceRepository(com.webank.wedatasphere.qualitis.dao.repository.TaskDataSourceRepository) TaskTypeException(com.webank.wedatasphere.qualitis.exception.TaskTypeException) TaskResult(com.webank.wedatasphere.qualitis.entity.TaskResult) ArgumentException(com.webank.wedatasphere.qualitis.exception.ArgumentException) RuleVariableNotFoundException(com.webank.wedatasphere.qualitis.exception.RuleVariableNotFoundException) AbstractJobSubmitter(com.webank.wedatasphere.qualitis.client.AbstractJobSubmitter) Logger(org.slf4j.Logger) RuleMetric(com.webank.wedatasphere.qualitis.entity.RuleMetric) TaskRuleSimpleRepository(com.webank.wedatasphere.qualitis.dao.repository.TaskRuleSimpleRepository) IOException(java.io.IOException) FastDateFormat(org.apache.commons.lang3.time.FastDateFormat) TableStatisticsInfo(com.webank.wedatasphere.qualitis.metadata.response.table.TableStatisticsInfo) FileOutputNameEnum(com.webank.wedatasphere.qualitis.rule.constant.FileOutputNameEnum) TaskRuleSimple(com.webank.wedatasphere.qualitis.entity.TaskRuleSimple) Component(org.springframework.stereotype.Component) ApplicationDao(com.webank.wedatasphere.qualitis.dao.ApplicationDao) DataQualityTask(com.webank.wedatasphere.qualitis.bean.DataQualityTask) UnitTransfer(com.webank.wedatasphere.qualitis.rule.util.UnitTransfer) TaskExecuteLimitConfig(com.webank.wedatasphere.qualitis.config.TaskExecuteLimitConfig) TaskResultDao(com.webank.wedatasphere.qualitis.dao.TaskResultDao) TaskDataSource(com.webank.wedatasphere.qualitis.entity.TaskDataSource) Task(com.webank.wedatasphere.qualitis.entity.Task) DataQualityTask(com.webank.wedatasphere.qualitis.bean.DataQualityTask) ArrayList(java.util.ArrayList) JobSubmitResult(com.webank.wedatasphere.qualitis.bean.JobSubmitResult) ClusterInfoNotConfigException(com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException) DataQualityTask(com.webank.wedatasphere.qualitis.bean.DataQualityTask) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) List(java.util.List) ArrayList(java.util.ArrayList) TaskRule(com.webank.wedatasphere.qualitis.bean.TaskRule) Rule(com.webank.wedatasphere.qualitis.rule.entity.Rule) DataQualityJob(com.webank.wedatasphere.qualitis.bean.DataQualityJob) TaskSubmitResult(com.webank.wedatasphere.qualitis.bean.TaskSubmitResult)

Example 3 with ClusterInfo

use of com.webank.wedatasphere.qualitis.entity.ClusterInfo in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method killApplication.

@Override
public GeneralResponse<?> killApplication(Application applicationInDb, String user) throws JobKillException, UnExpectedRequestException, ClusterInfoNotConfigException {
    List<Task> tasks = taskDao.findByApplication(applicationInDb);
    List<JobKillResult> results = new ArrayList<>();
    if (tasks == null || tasks.isEmpty()) {
        throw new UnExpectedRequestException("Sub tasks {&CAN_NOT_BE_NULL_OR_EMPTY}");
    }
    for (Task task : tasks) {
        ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(task.getClusterName());
        if (clusterInfo == null) {
            throw new ClusterInfoNotConfigException("Failed to find cluster id: " + task.getClusterName() + " configuration");
        }
        results.add(abstractJobSubmitter.killJob(user, clusterInfo.getClusterName(), task));
        task.setStatus(TaskStatusEnum.CANCELLED.getCode());
        task.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
        taskDao.save(task);
    }
    return new GeneralResponse<>("200", "{&SUCCESS_TO_KILL_TASK}", results.size());
}
Also used : GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) Task(com.webank.wedatasphere.qualitis.entity.Task) DataQualityTask(com.webank.wedatasphere.qualitis.bean.DataQualityTask) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) JobKillResult(com.webank.wedatasphere.qualitis.bean.JobKillResult) ArrayList(java.util.ArrayList) Date(java.util.Date) ClusterInfoNotConfigException(com.webank.wedatasphere.qualitis.exception.ClusterInfoNotConfigException)

Example 4 with ClusterInfo

use of com.webank.wedatasphere.qualitis.entity.ClusterInfo in project Qualitis by WeBankFinTech.

the class LinkisConfiguration method saveFullTree.

public Map saveFullTree(String clusterName, String creator, List<Map> fullTreeQueueName, List<Map> fullTree, String userName) throws UnExpectedRequestException {
    ClusterInfo clusterInfoInDb = clusterInfoDao.findByClusterName(clusterName);
    if (clusterInfoInDb == null) {
        throw new UnExpectedRequestException("cluster name {&ALREADY_EXIST}");
    }
    String url = UriBuilder.fromUri(clusterInfoInDb.getLinkisAddress()).path(linkisConfig.getPrefix()).path(linkisConfig.getSaveFullTree()).toString();
    Gson gson = new Gson();
    Map<String, Object> map = new HashMap<>(2);
    map.put("creator", creator);
    map.put("fullTree", fullTree);
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", userName);
    headers.add("Token-Code", clusterInfoInDb.getLinkisToken());
    HttpEntity entity = new HttpEntity<>(gson.toJson(map), headers);
    Map response = null;
    Map responseQueueName = null;
    LOGGER.info("Start to save configuration to linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.POST, entity);
    try {
        response = restTemplate.exchange(url, HttpMethod.POST, entity, Map.class).getBody();
        LOGGER.info("Finish to save configuration to linkis. response: {}", response);
        Integer code = (Integer) response.get("status");
        if (code != 0) {
            throw new UnExpectedRequestException("Failed to get configuration from linkis.");
        }
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        LOGGER.info("Failed to get configuration from linkis.");
        throw new UnExpectedRequestException("Failed to get configuration from linkis.");
    }
    return (Map) response.get("data");
}
Also used : UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) HttpHeaders(org.springframework.http.HttpHeaders) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) HttpEntity(org.springframework.http.HttpEntity) HashMap(java.util.HashMap) Gson(com.google.gson.Gson) HashMap(java.util.HashMap) Map(java.util.Map) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException)

Example 5 with ClusterInfo

use of com.webank.wedatasphere.qualitis.entity.ClusterInfo in project Qualitis by WeBankFinTech.

the class LinkisConfiguration method getFullTree.

public Map getFullTree(String clusterName, String user) throws UnExpectedRequestException {
    ClusterInfo clusterInfoInDb = clusterInfoDao.findByClusterName(clusterName);
    if (clusterInfoInDb == null) {
        throw new UnExpectedRequestException("cluster name {&DOES_NOT_EXIST}");
    }
    String url = UriBuilder.fromUri(clusterInfoInDb.getLinkisAddress()).path(linkisConfig.getPrefix()).path(linkisConfig.getGetFullTree()).queryParam("creator", linkisConfig.getAppName()).queryParam("engineType", "spark").queryParam("version", "2.4.3").toString();
    String urlQueueName = UriBuilder.fromUri(clusterInfoInDb.getLinkisAddress()).path(linkisConfig.getPrefix()).path(linkisConfig.getGetFullTree()).queryParam("creator", linkisConfig.getAppName()).toString();
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", user);
    headers.add("Token-Code", clusterInfoInDb.getLinkisToken());
    HttpEntity entity = new HttpEntity<>(headers);
    HttpEntity entityQueueName = new HttpEntity<>(headers);
    Map response = null;
    Map responseQueueName = null;
    LOGGER.info("Start to get configuration from linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
    try {
        response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
        responseQueueName = restTemplate.exchange(urlQueueName, HttpMethod.GET, entityQueueName, Map.class).getBody();
        LOGGER.info("Finish to get configuration from linkis. response: {}", response);
        Integer code = (Integer) response.get("status");
        if (code != 0) {
            throw new UnExpectedRequestException("Failed to get configuration from linkis.");
        }
        Integer codeQueueName = (Integer) responseQueueName.get("status");
        if (codeQueueName != 0) {
            throw new UnExpectedRequestException("Failed to get configuration from linkis.");
        }
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        LOGGER.info("Failed to get configuration from linkis.");
    }
    Map<String, Map> responseMap = new HashMap<>(2);
    responseMap.put("fule_tree", (Map) response.get("data"));
    responseMap.put("full_tree_queue_name", (Map) responseQueueName.get("data"));
    return responseMap;
}
Also used : UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) HttpHeaders(org.springframework.http.HttpHeaders) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) HttpEntity(org.springframework.http.HttpEntity) HashMap(java.util.HashMap) HashMap(java.util.HashMap) Map(java.util.Map) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException)

Aggregations

ClusterInfo (com.webank.wedatasphere.qualitis.entity.ClusterInfo)41 Map (java.util.Map)31 HttpEntity (org.springframework.http.HttpEntity)28 HttpHeaders (org.springframework.http.HttpHeaders)28 MetaDataAcquireFailedException (com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException)27 JSONObject (org.json.JSONObject)26 GeneralResponse (com.webank.wedatasphere.qualitis.response.GeneralResponse)25 ArrayList (java.util.ArrayList)18 UnExpectedRequestException (com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException)17 List (java.util.List)14 ColumnInfoDetail (com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail)7 UriBuilder (javax.ws.rs.core.UriBuilder)7 DataInfo (com.webank.wedatasphere.qualitis.metadata.response.DataInfo)6 Task (com.webank.wedatasphere.qualitis.entity.Task)5 PartitionStatisticsInfo (com.webank.wedatasphere.qualitis.metadata.response.table.PartitionStatisticsInfo)5 LinkisConfig (com.webank.wedatasphere.qualitis.config.LinkisConfig)4 ClusterInfoDao (com.webank.wedatasphere.qualitis.dao.ClusterInfoDao)4 TableStatisticsInfo (com.webank.wedatasphere.qualitis.metadata.response.table.TableStatisticsInfo)4 JSONException (org.json.JSONException)4 RestClientException (org.springframework.web.client.RestClientException)4