Examples with MetaDataAcquireFailedException com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException used on opensource projects

Example 1 with MetaDataAcquireFailedException

use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method submitApplication.

/**
 * Submit job to linkis
 */
@Override
public List<TaskSubmitResult> submitApplication(List<Rule> rules, String nodeName, String createTime, String user, String database, StringBuffer partition, Date date, Application application, String cluster, String startupParam, String setFlag, Map<String, String> execParams, StringBuffer runDate, Map<Long, Map> dataSourceMysqlConnect) throws ArgumentException, TaskTypeException, ConvertException, DataQualityTaskException, RuleVariableNotSupportException, RuleVariableNotFoundException, JobSubmitException, ClusterInfoNotConfigException, IOException, UnExpectedRequestException, MetaDataAcquireFailedException {
    String csId = rules.iterator().next().getCsId();
    // Check if cluster supported
    LOGGER.info("Start to collect rule to clusters");
    Map<String, List<Rule>> clusterNameMap = getRuleCluster(rules);
    LOGGER.info("Succeed to classify rules by cluster, cluster map: {}", clusterNameMap);
    if (StringUtils.isNotBlank(cluster)) {
        LOGGER.info("When pick up a cluster, these datasources of rules must be from one cluster. Now start to put into the specify cluster.\n");
        putAllRulesIntoSpecifyCluster(clusterNameMap, cluster);
        LOGGER.info("Success to put into the specify cluster.\n");
    }
    List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
    for (String clusterName : clusterNameMap.keySet()) {
        List<Rule> clusterRules = clusterNameMap.get(clusterName);
        if (StringUtils.isNotBlank(cluster)) {
            clusterName = cluster;
        }
        ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
        LOGGER.info("Start to check cluster config.");
        if (clusterInfo == null) {
            throw new ClusterInfoNotConfigException(clusterName + " {&DOES_NOT_EXIST}");
        }
        LOGGER.info("Succeed to pass the check of cluster config. All cluster of rules are configured");
        // Divide rule into tasks
        List<DataQualityTask> tasks = TaskDividerFactory.getDivider().divide(clusterRules, application.getId(), createTime, partition.toString(), date, database, user, taskExecuteLimitConfig.getTaskExecuteRuleSize());
        LOGGER.info("Succeed to divide application into tasks. result: {}", tasks);
        // Save divided tasks
        saveDividedTask(tasks, clusterInfo, rules, application, createTime);
        // Convert tasks into job
        List<DataQualityJob> jobList = new ArrayList<>();
        for (DataQualityTask task : tasks) {
            DataQualityJob job = templateConverterFactory.getConverter(task).convert(task, date, setFlag, execParams, runDate.toString(), clusterInfo.getClusterType(), dataSourceMysqlConnect);
            job.setUser(task.getUser());
            jobList.add(job);
            List<Long> ruleIdList = task.getRuleTaskDetails().stream().map(r -> r.getRule().getId()).collect(Collectors.toList());
            LOGGER.info("Succeed to convert rule_id: {} into code. code: {}", ruleIdList, job.getJobCode());
        }
        LOGGER.info("Succeed to convert all template into codes. codes: {}", jobList);
        // Submit job to linkis
        List<JobSubmitResult> submitResults = new ArrayList<>();
        for (DataQualityJob job : jobList) {
            String code = String.join("\n", job.getJobCode());
            String proxy = job.getUser();
            Long taskId = job.getTaskId();
            // Compatible with new and old submission interfaces.
            JobSubmitResult result = null;
            boolean engineReUse = false;
            if (StringUtils.isNotBlank(startupParam)) {
                String[] startupParams = startupParam.split(SpecCharEnum.DIVIDER.getValue());
                for (String param : startupParams) {
                    if (StringUtils.isEmpty(param)) {
                        continue;
                    }
                    String[] paramStrs = param.split("=");
                    if (paramStrs.length < 2) {
                        continue;
                    }
                    String key = paramStrs[0];
                    String value = paramStrs[1];
                    if ("engine_reuse".equals(key)) {
                        if ("true".equals(value)) {
                            engineReUse = true;
                            startupParam = startupParam.replace("engine_reuse=true", "");
                        } else {
                            engineReUse = false;
                            startupParam = startupParam.replace("engine_reuse=false", "");
                        }
                        break;
                    }
                }
            }
            if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
                result = abstractJobSubmitter.submitJobNew(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam(), engineReUse);
            } else {
                result = abstractJobSubmitter.submitJob(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam());
            }
            if (result != null) {
                submitResults.add(result);
            } else {
                Task taskInDb = taskDao.findById(taskId);
                taskInDb.setStatus(TaskStatusEnum.TASK_NOT_EXIST.getCode());
                taskDao.save(taskInDb);
                taskSubmitResults.add(new TaskSubmitResult(application.getId(), null, clusterInfo.getClusterName()));
            }
        }
        // Rewrite task remote ID.
        rewriteTaskRemoteInfo(submitResults, taskSubmitResults, application.getId(), clusterInfo.getClusterName());
    }
    return taskSubmitResults;
}

Example 2 with MetaDataAcquireFailedException

use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.

the class MetaDataClientImpl method getColumnInfo.

@Override
public List<ColumnInfoDetail> getColumnInfo(String clusterName, String dbName, String tableName, String userName) throws MetaDataAcquireFailedException, UnExpectedRequestException {
    ClusterInfo clusterInfo = checkClusterNameExists(clusterName);
    // send request to get table comment.
    String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getColumnInfo()).queryParam("database", dbName).queryParam("tableName", tableName).toString();
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", userName);
    headers.add("Token-Code", clusterInfo.getLinkisToken());
    HttpEntity<Object> entity = new HttpEntity<>(headers);
    LOGGER.info("Start to get column info by user and cluster and db and table by linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
    Map<String, Object> response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
    LOGGER.info("Finished to get column info by user and cluster and db and table by linkis. response: {}", response);
    if (!checkResponse(response)) {
        String message = (String) response.get("message");
        LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
        throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
    }
    List<Map<String, Object>> tableFieldInfo = (List<Map<String, Object>>) ((Map<String, Object>) response.get("data")).get("tableFieldsInfo");
    List<ColumnInfoDetail> result = new ArrayList<>();
    for (Map<String, Object> map : tableFieldInfo) {
        ColumnInfoDetail columnInfoDetail = new ColumnInfoDetail();
        columnInfoDetail.setFieldName(map.get("name").toString());
        columnInfoDetail.setDataType(map.get("type").toString());
        if (map.get("length") != null && !"".equals(map.get("length").toString())) {
            columnInfoDetail.setColumnLen(Integer.parseInt(map.get("length").toString()));
        }
        columnInfoDetail.setColumnAlias(map.get("alias") == null ? "" : map.get("alias").toString());
        columnInfoDetail.setColumnComment(map.get("comment") == null ? "" : map.get("comment").toString());
        columnInfoDetail.setPrimary((Boolean) map.get("primary"));
        columnInfoDetail.setPartitionField((Boolean) map.get("partitionField"));
        result.add(columnInfoDetail);
    }
    return result;
}

Example 3 with MetaDataAcquireFailedException

use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.

the class MetaDataClientImpl method getDataSourceVersions.

@Override
public GeneralResponse<Map> getDataSourceVersions(String clusterName, String authUser, Long dataSourceId) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    // Check existence of cluster name
    ClusterInfo clusterInfo = checkClusterNameExists(clusterName);
    // send request to get dbs
    String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getDatasourceVersions()).toString().replace("{DATA_SOURCE_ID}", dataSourceId.toString());
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", authUser);
    headers.add("Token-Code", clusterInfo.getLinkisToken());
    HttpEntity<Object> entity = new HttpEntity<>(headers);
    LOGGER.info("Start to get data source versions by user and cluster by linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
    Map<String, Object> response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
    LOGGER.info("Finish to get data source versions by user and cluster by linkis. response: {}", response);
    if (!checkResponse(response)) {
        String message = (String) response.get("message");
        LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
        throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
    }
    Map data = (Map) response.get(LinkisResponseKeyEnum.DATA.getKey());
    List<Map> types = (List<Map>) data.get("versions");
    return new GeneralResponse<>("200", "Success to get datasource version", data);
}

Example 4 with MetaDataAcquireFailedException

use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.

the class MetaDataClientImpl method getTablesByDataSource.

@Override
public Map getTablesByDataSource(String clusterName, String authUser, Long dataSourceId, String dbName) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    // Check existence of cluster name
    ClusterInfo clusterInfo = checkClusterNameExists(clusterName);
    // send request to get dbs
    String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getDatasourceTable()).queryParam("system", "Qualitis").toString().replace("{DATA_SOURCE_ID}", dataSourceId.toString()).replace("{DATA_SOURCE_DB}", dbName);
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", authUser);
    headers.add("Token-Code", clusterInfo.getLinkisToken());
    HttpEntity<Object> entity = new HttpEntity<>(headers);
    LOGGER.info("Start to get tables by data source. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
    Map<String, Object> response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
    LOGGER.info("Finish to get tables by data source. response: {}", response);
    if (!checkResponse(response)) {
        String message = (String) response.get("message");
        LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
        throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
    }
    Map data = (Map) response.get(LinkisResponseKeyEnum.DATA.getKey());
    List<Map> tables = (List<Map>) data.get("tables");
    if (CollectionUtils.isEmpty(tables)) {
        LOGGER.info("No tables with data source to be choosed.");
    }
    return data;
}

Example 5 with MetaDataAcquireFailedException

use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.

the class MetaDataClientImpl method getPartitionStatisticsInfo.

@Override
public PartitionStatisticsInfo getPartitionStatisticsInfo(String clusterName, String dbName, String tableName, String partitionPath, String userName) throws UnExpectedRequestException, MetaDataAcquireFailedException, RestClientException {
    ClusterInfo clusterInfo = checkClusterNameExists(clusterName);
    // send request to get dbs
    String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getPartitionStatistics()).queryParam("database", dbName).queryParam("tableName", tableName).queryParam("partitionPath", partitionPath).toString();
    try {
        url = URLDecoder.decode(url, "UTF-8");
    } catch (UnsupportedEncodingException e) {
        LOGGER.error(e.getMessage(), e);
        throw new UnExpectedRequestException("Decode get partition statistic info exception", 500);
    }
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", userName);
    headers.add("Token-Code", clusterInfo.getLinkisToken());
    HttpEntity<Object> entity = new HttpEntity<>(headers);
    LOGGER.info("Start to get partition info by linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
    Map<String, Object> response = null;
    try {
        response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
    } catch (ResourceAccessException e) {
        LOGGER.error(e.getMessage(), e);
        throw new MetaDataAcquireFailedException("Error! Can not get partition info from linkis, exception: " + e.getMessage(), 500);
    }
    LOGGER.info("Finish to get partition info by linkis. response: {}", response);
    if (!checkResponse(response)) {
        String message = (String) response.get("message");
        LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
        throw new MetaDataAcquireFailedException("Error! Can not get partition info from linkis, exception: " + message);
    }
    Map<String, Object> result = (Map<String, Object>) ((Map<String, Object>) response.get("data")).get("partitionStatisticInfo");
    PartitionStatisticsInfo partitionStatisticsInfo = new PartitionStatisticsInfo();
    partitionStatisticsInfo.setPartitionChildCount(Integer.parseInt(result.get("fileNum").toString()));
    partitionStatisticsInfo.setPartitionSize(result.get("partitionSize").toString());
    partitionStatisticsInfo.setPartitions((List<Map>) result.get("childrens"));
    return partitionStatisticsInfo;
}