Examples with MetaDataAcquireFailedException com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException used on opensource projects

Example 16 with MetaDataAcquireFailedException

use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.

the class MetaDataController method getTablesByDataSource.

@GET
@Path("data_source/tables")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public GeneralResponse<GetAllResponse<TableInfoDetail>> getTablesByDataSource(@QueryParam("clusterName") String clusterName, @QueryParam("proxyUser") String proxyUser, @QueryParam("dataSourceId") Long dataSourceId, @QueryParam("dbName") String dbName) {
    try {
        if (dataSourceId == null || StringUtils.isBlank(dbName) || StringUtils.isBlank(clusterName)) {
            throw new UnExpectedRequestException("Request " + "{&CAN_NOT_BE_NULL_OR_EMPTY}");
        }
        Map response = metaDataService.getTablesByDataSource(clusterName, proxyUser, dataSourceId, dbName);
        GetAllResponse allResponse = new GetAllResponse();
        List<String> tables = (List<String>) response.get("tables");
        List<TableInfoDetail> tableInfoDetails = new ArrayList<>(CollectionUtils.isEmpty(tables) ? 0 : tables.size());
        for (String table : tables) {
            TableInfoDetail tableInfoDetail = new TableInfoDetail(table);
            tableInfoDetails.add(tableInfoDetail);
        }
        allResponse.setTotal(CollectionUtils.isEmpty(tableInfoDetails) ? 0 : tableInfoDetails.size());
        allResponse.setData(tableInfoDetails);
        return new GeneralResponse<>("200", "{&GET_TABLE_SUCCESSFULLY}", allResponse);
    } catch (MetaDataAcquireFailedException e) {
        LOGGER.error(e.getMessage(), e);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (UnExpectedRequestException e) {
        LOGGER.error(e.getMessage(), e);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        return new GeneralResponse<>("500", "{&FAILED_TO_GET_TABLE_BY_DATABASE}", null);
    }
}

Example 17 with MetaDataAcquireFailedException

use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.

the class RuleQueryServiceImpl method getColumnsByTableName.

@Override
public List<ColumnInfoDetail> getColumnsByTableName(String clusterName, Long datasourceId, String dbName, String tableName, String userName) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    List<ColumnInfoDetail> result = null;
    User user = userDao.findByUsername(userName);
    if (user == null) {
        throw new UnExpectedRequestException(String.format("{&FAILED_TO_FIND_USER} %s", userName));
    }
    try {
        if (StringUtils.isBlank(dbName)) {
            List<Rule> rules = ruleDataSourceDao.findRuleByDataSource(clusterName, dbName, tableName, "%", userName);
            if (rules == null || rules.isEmpty()) {
                LOGGER.info("No rules for this datasource!");
                return null;
            }
            LOGGER.info("Rules related with context service table: [{}] are: {}", tableName, rules.toArray());
            String csId = rules.get(0).getCsId();
            String nodeName = rules.get(0).getName();
            GetUserTableByCsIdRequest getUserTableByCsIdRequest = new GetUserTableByCsIdRequest();
            getUserTableByCsIdRequest.setClusterName(clusterName);
            getUserTableByCsIdRequest.setLoginUser(userName);
            getUserTableByCsIdRequest.setNodeName(nodeName);
            getUserTableByCsIdRequest.setCsId(csId);
            DataInfo<CsTableInfoDetail> csTableInfoDetails = metaDataClient.getTableByCsId(getUserTableByCsIdRequest);
            if (csTableInfoDetails.getTotalCount() == 0) {
                LOGGER.info("Cannot find context service table with existed rules!");
                return null;
            }
            for (CsTableInfoDetail csTableInfoDetail : csTableInfoDetails.getContent()) {
                if (csTableInfoDetail.getTableName().equals(tableName)) {
                    GetUserColumnByCsRequest getUserColumnByCsRequest = new GetUserColumnByCsRequest();
                    getUserColumnByCsRequest.setClusterName(clusterName);
                    getUserColumnByCsRequest.setContextKey(csTableInfoDetail.getContextKey());
                    getUserColumnByCsRequest.setCsId(csId);
                    getUserColumnByCsRequest.setLoginUser(userName);
                    result = metaDataClient.getColumnByCsId(getUserColumnByCsRequest).getContent();
                    setRuleCount(result, user.getId(), clusterName, dbName, tableName);
                    break;
                } else {
                    continue;
                }
            }
        } else {
            if (datasourceId == null) {
                result = metaDataClient.getColumnInfo(clusterName, dbName, tableName, userName);
            } else {
                result = metaDataClient.getColumnsByDataSource(clusterName, userName, datasourceId, dbName, tableName).getContent();
            }
            setRuleCount(result, user.getId(), clusterName, dbName, tableName);
        }
        LOGGER.info("Datasource table number of columns: {}", result == null ? 0 : result.size());
    } catch (MetaDataAcquireFailedException e) {
        LOGGER.error("Datasource colums info {&DOES_NOT_EXIST}. Exception: {}", e);
        throw new MetaDataAcquireFailedException("Datasource colums info {&DOES_NOT_EXIST}. Exception: " + e.getMessage());
    } catch (UnExpectedRequestException e) {
        LOGGER.error("{&QUERY_PARAM_HAS_ERROR}. Exception: {}", e);
        throw new UnExpectedRequestException("{&QUERY_PARAM_HAS_ERROR}. Exception: " + e.getMessage());
    } catch (Exception e) {
        LOGGER.error("exception: {}", e);
    }
    return result;
}

Example 18 with MetaDataAcquireFailedException

use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.

the class MetaDataController method getDbsByDataSource.

@GET
@Path("data_source/dbs")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public GeneralResponse<GetAllResponse<DbInfoDetail>> getDbsByDataSource(@QueryParam("clusterName") String clusterName, @QueryParam("proxyUser") String proxyUser, @QueryParam("dataSourceId") Long dataSourceId) {
    try {
        if (dataSourceId == null || StringUtils.isBlank(clusterName)) {
            throw new UnExpectedRequestException("Request " + "{&CAN_NOT_BE_NULL_OR_EMPTY}");
        }
        Map response = metaDataService.getDbsByDataSource(clusterName, proxyUser, dataSourceId);
        GetAllResponse allResponse = new GetAllResponse();
        List<String> dbs = (List<String>) response.get("dbs");
        List<DbInfoDetail> dbInfoDetails = new ArrayList<>(CollectionUtils.isEmpty(dbs) ? 0 : dbs.size());
        for (String db : dbs) {
            DbInfoDetail dbInfoDetail = new DbInfoDetail(db);
            dbInfoDetails.add(dbInfoDetail);
        }
        allResponse.setTotal(CollectionUtils.isEmpty(dbInfoDetails) ? 0 : dbInfoDetails.size());
        allResponse.setData(dbInfoDetails);
        return new GeneralResponse<>("200", "{&GET_DB_SUCCESSFULLY}", allResponse);
    } catch (MetaDataAcquireFailedException e) {
        LOGGER.error(e.getMessage(), e);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (UnExpectedRequestException e) {
        LOGGER.error(e.getMessage(), e);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        return new GeneralResponse<>("500", "{&FAILED_TO_GET_DATABASE_BY_CLUSTER}", null);
    }
}

Example 19 with MetaDataAcquireFailedException

use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method submitRules.

/**
 * Generate jobs by rules and submit jobs to linkis.
 *
 * @param ruleIds
 * @param partition
 * @param createUser
 * @param executionUser
 * @param nodeName
 * @param projectId
 * @param ruleGroupId
 * @param execParams
 * @param runDate
 * @param invokeCode
 * @return
 * @throws UnExpectedRequestException
 */
@Override
@Transactional(propagation = Propagation.REQUIRES_NEW, rollbackFor = { RuntimeException.class, UnExpectedRequestException.class })
public GeneralResponse<?> submitRules(List<Long> ruleIds, StringBuffer partition, String createUser, String executionUser, String nodeName, Long projectId, Long ruleGroupId, String startupParam, String clusterName, String setFlag, Map<String, String> execParams, String execParamStr, StringBuffer runDate, Integer invokeCode) {
    // Get rule eneity.
    List<Rule> rules = ruleDao.findByIds(ruleIds);
    // Init application basic info.
    Date date = new Date();
    Application newApplication = outerExecutionService.generateApplicationInfo(createUser, executionUser, date, invokeCode);
    newApplication.setProjectId(projectId);
    newApplication.setRuleGroupId(ruleGroupId);
    newApplication.setPartition(partition.toString());
    newApplication.setClusterName(clusterName);
    newApplication.setStartupParam(startupParam);
    newApplication.setExecutionParam(execParamStr);
    newApplication.setSetFlag(setFlag);
    ApplicationTaskSimpleResponse response;
    try {
        response = outerExecutionService.commonExecution(rules, partition, executionUser, nodeName, startupParam, clusterName, setFlag, execParams, newApplication, date, runDate);
    } catch (BothNullDatasourceException e) {
        catchAndSolve(e, ApplicationCommentEnum.BOTH_NULL_ISSUES.getCode(), ApplicationStatusEnum.FINISHED.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (LeftNullDatasourceException e) {
        catchAndSolve(e, ApplicationCommentEnum.LEFT_NULL_DATA_ISSUES.getCode(), ApplicationStatusEnum.NOT_PASS.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (RightNullDatasourceException e) {
        catchAndSolve(e, ApplicationCommentEnum.RIGHT_NULL_DATA_ISSUES.getCode(), ApplicationStatusEnum.NOT_PASS.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (MetaDataAcquireFailedException e) {
        catchAndSolve(e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", "{&THE_CHECK_FIELD_HAS_BEEN_MODIFIED}", null);
    } catch (DataSourceMoveException e) {
        catchAndSolve(e, ApplicationCommentEnum.PERMISSION_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (DataSourceOverSizeException e) {
        catchAndSolve(e, ApplicationCommentEnum.PERMISSION_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (ParseException e) {
        catchAndSolve(e, ApplicationCommentEnum.GRAMMAR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
        LOGGER.error(e.getMessage(), e);
        return new GeneralResponse<>("500", "{&PARSE_SQL_FAILED_PLEASE_CHECKOUT_YOUR_CUSTOM_SQL}", null);
    } catch (Exception e) {
        catchAndSolve(e, ApplicationCommentEnum.UNKNOWN_ERROR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules, newApplication);
        return new GeneralResponse<>("500", e.getMessage(), null);
    }
    LOGGER.info("Succeed to dispatch task. response: {}", response);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_DISPATCH_TASK}", response);
}

Example 20 with MetaDataAcquireFailedException

use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method submitRulesWithDynamicPartition.

private void submitRulesWithDynamicPartition(List<ApplicationSubmitRequest> applicationSubmitRequests, Long projectId, Long ruleGroupId, List<Rule> rules, List<Long> ruleIds, String executionUser, boolean dynamicPartition, String clusterName, StringBuffer partition, String dynamicPartitionPrefix) throws UnExpectedRequestException, NoPartitionException {
    if (dynamicPartition) {
        for (Rule rule : rules) {
            RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
            if (ruleDataSource == null) {
                throw new UnExpectedRequestException("Rule datasource has been broken.");
            }
            if (RuleTypeEnum.CUSTOM_RULE.getCode().equals(rule.getRuleType())) {
                throw new UnExpectedRequestException(RuleTypeEnum.CUSTOM_RULE.getMessage() + " {&IS_NOT_SUPPORT}");
            }
            List<Map> currentPartitionMap;
            try {
                TableStatisticsInfo tableStatisticsInfo = metaDataClient.getTableStatisticsInfo(StringUtils.isBlank(clusterName) ? ruleDataSource.getClusterName() : clusterName, ruleDataSource.getDbName(), ruleDataSource.getTableName(), executionUser);
                currentPartitionMap = tableStatisticsInfo.getPartitions();
                if (StringUtils.isNotEmpty(dynamicPartitionPrefix)) {
                    metaDataClient.getPartitionStatisticsInfo(StringUtils.isBlank(clusterName) ? ruleDataSource.getClusterName() : clusterName, ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(dynamicPartitionPrefix), executionUser);
                }
            } catch (MetaDataAcquireFailedException e) {
                LOGGER.error("Dynamic submit failed.", e);
                throw new NoPartitionException(ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + "{&HAS_NO_PARTITIONS_TO_BE_EXECUTE}");
            }
            List<String> partitionList = new ArrayList<>();
            if (CollectionUtils.isNotEmpty(currentPartitionMap)) {
                ruleIds.remove(rule.getId());
                getPartitionListAndSubmit(currentPartitionMap, partitionList, rule, partition, filterToPartitionPath(dynamicPartitionPrefix), applicationSubmitRequests, projectId, ruleGroupId);
            } else {
                throw new NoPartitionException(ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + "{&HAS_NO_PARTITIONS_TO_BE_EXECUTE}");
            }
        }
    }
}