Examples with ColumnInfoDetail com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail used on opensource projects

Example 6 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class MetaDataServiceImpl method getUserColumnByTableId.

@Override
public GeneralResponse<GetAllResponse<ColumnInfoDetail>> getUserColumnByTableId(GetUserColumnByTableIdRequest request) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    // Check Arguments
    checkRequest(request);
    // Get login user
    String userName = "";
    if (request.getLoginUser() != null) {
        userName = request.getLoginUser();
        LOGGER.info("Recover user[{}] from is get column info.", userName);
    } else {
        userName = HttpUtils.getUserName(httpServletRequest);
        ;
    }
    if (StringUtils.isNotBlank(request.getProxyUser())) {
        userName = request.getProxyUser();
    }
    GetColumnByUserAndTableRequest getColumnByUserAndTableRequest = new GetColumnByUserAndTableRequest(userName, request.getStartIndex(), request.getPageSize(), request.getClusterName(), request.getDbName(), request.getTableName());
    DataInfo<ColumnInfoDetail> response = metaDataClient.getColumnByUserAndTable(getColumnByUserAndTableRequest);
    GetAllResponse<ColumnInfoDetail> result = new GetAllResponse<>();
    result.setTotal(response.getTotalCount());
    result.setData(response.getContent());
    LOGGER.info("Succeed to get column by table. table: {}.{}.{}", request.getClusterName(), request.getDbName(), request.getTableName());
    return new GeneralResponse<>("200", "{&GET_COLUMN_SUCCESSFULLY}", result);
}

Example 7 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class RuleQueryServiceImpl method getColumnsByTableName.

@Override
public List<ColumnInfoDetail> getColumnsByTableName(String clusterName, Long datasourceId, String dbName, String tableName, String userName) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    List<ColumnInfoDetail> result = null;
    User user = userDao.findByUsername(userName);
    if (user == null) {
        throw new UnExpectedRequestException(String.format("{&FAILED_TO_FIND_USER} %s", userName));
    }
    try {
        if (StringUtils.isBlank(dbName)) {
            List<Rule> rules = ruleDataSourceDao.findRuleByDataSource(clusterName, dbName, tableName, "%", userName);
            if (rules == null || rules.isEmpty()) {
                LOGGER.info("No rules for this datasource!");
                return null;
            }
            LOGGER.info("Rules related with context service table: [{}] are: {}", tableName, rules.toArray());
            String csId = rules.get(0).getCsId();
            String nodeName = rules.get(0).getName();
            GetUserTableByCsIdRequest getUserTableByCsIdRequest = new GetUserTableByCsIdRequest();
            getUserTableByCsIdRequest.setClusterName(clusterName);
            getUserTableByCsIdRequest.setLoginUser(userName);
            getUserTableByCsIdRequest.setNodeName(nodeName);
            getUserTableByCsIdRequest.setCsId(csId);
            DataInfo<CsTableInfoDetail> csTableInfoDetails = metaDataClient.getTableByCsId(getUserTableByCsIdRequest);
            if (csTableInfoDetails.getTotalCount() == 0) {
                LOGGER.info("Cannot find context service table with existed rules!");
                return null;
            }
            for (CsTableInfoDetail csTableInfoDetail : csTableInfoDetails.getContent()) {
                if (csTableInfoDetail.getTableName().equals(tableName)) {
                    GetUserColumnByCsRequest getUserColumnByCsRequest = new GetUserColumnByCsRequest();
                    getUserColumnByCsRequest.setClusterName(clusterName);
                    getUserColumnByCsRequest.setContextKey(csTableInfoDetail.getContextKey());
                    getUserColumnByCsRequest.setCsId(csId);
                    getUserColumnByCsRequest.setLoginUser(userName);
                    result = metaDataClient.getColumnByCsId(getUserColumnByCsRequest).getContent();
                    setRuleCount(result, user.getId(), clusterName, dbName, tableName);
                    break;
                } else {
                    continue;
                }
            }
        } else {
            if (datasourceId == null) {
                result = metaDataClient.getColumnInfo(clusterName, dbName, tableName, userName);
            } else {
                result = metaDataClient.getColumnsByDataSource(clusterName, userName, datasourceId, dbName, tableName).getContent();
            }
            setRuleCount(result, user.getId(), clusterName, dbName, tableName);
        }
        LOGGER.info("Datasource table number of columns: {}", result == null ? 0 : result.size());
    } catch (MetaDataAcquireFailedException e) {
        LOGGER.error("Datasource colums info {&DOES_NOT_EXIST}. Exception: {}", e);
        throw new MetaDataAcquireFailedException("Datasource colums info {&DOES_NOT_EXIST}. Exception: " + e.getMessage());
    } catch (UnExpectedRequestException e) {
        LOGGER.error("{&QUERY_PARAM_HAS_ERROR}. Exception: {}", e);
        throw new UnExpectedRequestException("{&QUERY_PARAM_HAS_ERROR}. Exception: " + e.getMessage());
    } catch (Exception e) {
        LOGGER.error("exception: {}", e);
    }
    return result;
}

Example 8 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class AddMultiRuleRequestBuilder method solveDatasource.

private Map<String, List<ColumnInfoDetail>> solveDatasource(StringBuffer cluster, String datasource) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    MultiDataSourceConfigRequest source;
    MultiDataSourceConfigRequest target;
    List<MultiDataSourceJoinConfigRequest> mappings = new ArrayList<>();
    String[] datasourceStrs = datasource.split(SpecCharEnum.VERTICAL_BAR.getValue());
    String sourcStr = datasourceStrs[0];
    String targetStr = datasourceStrs[1];
    Matcher matcherName = DATA_SOURCE_NAME.matcher(cluster.toString().toUpperCase());
    Matcher matcherId = DATA_SOURCE_ID.matcher(cluster.toString().toUpperCase());
    String dataSourceName = "";
    String dataSourceId = "";
    while (matcherId.find()) {
        String group = matcherId.group();
        dataSourceId = group.replace(".(", "").replace(")", "").split("=")[1];
        int startIndex = cluster.toString().toUpperCase().indexOf(group);
        String replaceStr = cluster.substring(startIndex, startIndex + group.length());
        cluster.delete(startIndex, startIndex + group.length());
    }
    if (StringUtils.isBlank(dataSourceId)) {
        while (matcherName.find()) {
            String group = matcherName.group();
            int startIndex = cluster.toString().toUpperCase().indexOf(group);
            String replaceStr = cluster.substring(startIndex, startIndex + group.length());
            dataSourceName = replaceStr.replace(".(", "").replace(")", "").split("=")[1];
            cluster.delete(startIndex, startIndex + group.length());
        }
    }
    source = toRequest(cluster.toString(), sourcStr, dataSourceId, dataSourceName);
    target = toRequest(cluster.toString(), targetStr, dataSourceId, dataSourceName);
    addMultiSourceRuleRequest.setSource(source);
    addMultiSourceRuleRequest.setTarget(target);
    addMultiSourceRuleRequest.setMappings(mappings);
    if (datasourceStrs.length > TWO) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    List<ColumnInfoDetail> leftCols = metaDataClient.getColumnInfo(cluster.toString(), source.getDbName(), source.getTableName(), userName);
    List<ColumnInfoDetail> rightCols = metaDataClient.getColumnInfo(cluster.toString(), target.getDbName(), target.getTableName(), userName);
    Map<String, List<ColumnInfoDetail>> colsMap = new HashMap<>(2);
    colsMap.put(TMP_1, leftCols);
    colsMap.put(TMP_2, rightCols);
    return colsMap;
}

Example 9 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method checkDatasource.

private void checkDatasource(Rule currentRule, String userName, StringBuffer partition, List<Map<String, String>> mappingCols, String nodeName, String clusterName, Map<Long, Map> dataSourceMysqlConnect) throws UnExpectedRequestException, MetaDataAcquireFailedException, DataSourceOverSizeException, DataSourceMoveException, BothNullDatasourceException, LeftNullDatasourceException, RightNullDatasourceException {
    // For multi source rule to check tables' size before submit.
    List<Double> datasourceSizeList = new ArrayList<>(currentRule.getRuleDataSources().size());
    for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
        Map<String, String> mappingCol = null;
        if (ORIGINAL_INDEX.equals(ruleDataSource.getDatasourceIndex())) {
            continue;
        }
        if (ruleDataSource.getDatasourceIndex() != null && mappingCols.get(ruleDataSource.getDatasourceIndex()).size() > 0) {
            mappingCol = mappingCols.get(ruleDataSource.getDatasourceIndex());
        }
        if (ruleDataSource.getLinkisDataSourceId() != null) {
            LOGGER.info("Start to solve relationship datasource info.");
            checkRdmsSqlMetaInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource, mappingCol);
            GeneralResponse<Map> dataSourceInfoDetail = metaDataClient.getDataSourceInfoDetail(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource.getLinkisDataSourceId(), ruleDataSource.getLinkisDataSourceVersionId());
            GeneralResponse<Map> dataSourceConnectParams = metaDataClient.getDataSourceConnectParams(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource.getLinkisDataSourceId(), ruleDataSource.getLinkisDataSourceVersionId());
            Map connectParamsReal = (Map) dataSourceConnectParams.getData().get("connectParams");
            if (connectParamsReal.size() == 0) {
                throw new UnExpectedRequestException("{&THE_DATASOURCE_IS_NOT_DEPLOYED}");
            }
            Map connectParams = (Map) ((Map) dataSourceInfoDetail.getData().get("info")).get("connectParams");
            String dataType = (String) ((Map) ((Map) dataSourceInfoDetail.getData().get("info")).get("dataSourceType")).get("name");
            connectParams.put("dataType", dataType);
            dataSourceMysqlConnect.put(ruleDataSource.getId(), connectParams);
            continue;
        }
        // Parse filter fields.
        List<String> filterFields = getFilterFields(partition.toString());
        if (StringUtils.isNotBlank(ruleDataSource.getDbName()) && !ruleDataSource.getDbName().equals(RuleConstraintEnum.CUSTOM_DATABASE_PREFIS.getValue())) {
            // Get actual fields info.
            List<ColumnInfoDetail> cols = metaDataClient.getColumnInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), userName);
            if (CollectionUtils.isEmpty(cols)) {
                throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "]. {&RULE_DATASOURCE_BE_MOVED}");
            }
            // Get actual partition fields.
            List<String> partitionFields = cols.stream().filter(ColumnInfoDetail::getPartitionField).map(ColumnInfoDetail::getFieldName).collect(Collectors.toList());
            // Check filter fields.
            boolean partitionTable = CollectionUtils.isNotEmpty(partitionFields);
            if (partitionTable && partition.length() > 0) {
                for (String filter : filterFields) {
                    if (!partitionFields.contains(filter)) {
                        throw new UnExpectedRequestException("Table[" + ruleDataSource.getTableName() + "]. {&THE_CHECK_FIELD_DOES_NOT_EXIST_IN_PARTITIONS}[" + filter + "]");
                    }
                }
                // Check partition size.
                PartitionStatisticsInfo partitionStatisticsInfo = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(partition.toString()), userName);
                String fullSize = partitionStatisticsInfo.getPartitionSize();
                ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
                if (clusterInfo != null && StringUtils.isNotBlank(clusterInfo.getSkipDataSize()) && StringUtils.isNotBlank(fullSize)) {
                    double number = 0;
                    String unit = "B";
                    if (!"0B".equals(fullSize)) {
                        number = Double.parseDouble(fullSize.split(" ")[0]);
                        unit = fullSize.split(" ")[1];
                    }
                    datasourceSizeList.add(number);
                    String[] skipDataSize = clusterInfo.getSkipDataSize().split(" ");
                    double res = UnitTransfer.alarmconfigToTaskResult(number, skipDataSize[1], unit);
                    LOGGER.info("Check datasource[" + fullSize + "] if or not oversize with system config[" + clusterInfo.getSkipDataSize() + "]");
                    if (res > Double.parseDouble(skipDataSize[0])) {
                        throw new DataSourceOverSizeException("Table[" + ruleDataSource.getTableName() + "]. {&TABLE_IS_OVERSIZE_WITH_SYSTEM_CONFIG}:[" + clusterInfo.getSkipDataSize() + "]");
                    }
                }
            } else {
                // Check table size.
                TableStatisticsInfo tableStatisticsInfo = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), userName);
                String fullSize = tableStatisticsInfo.getTableSize();
                if (NULL_TABLE_SIZE.equals(fullSize)) {
                    throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "] {&RULE_DATASOURCE_BE_MOVED}");
                }
                ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
                if (clusterInfo != null && StringUtils.isNotBlank(clusterInfo.getSkipDataSize()) && StringUtils.isNotBlank(fullSize)) {
                    LOGGER.info("Check datasource[" + fullSize + "] if or not oversize with system config[" + clusterInfo.getSkipDataSize() + "]");
                    double number = 0;
                    String unit = "B";
                    if (!"0B".equals(fullSize)) {
                        number = Double.parseDouble(fullSize.split(" ")[0]);
                        unit = fullSize.split(" ")[1];
                    }
                    datasourceSizeList.add(number);
                    String[] skipDataSize = clusterInfo.getSkipDataSize().split(" ");
                    double res = UnitTransfer.alarmconfigToTaskResult(number, skipDataSize[1], unit);
                    if (res > Double.parseDouble(skipDataSize[0])) {
                        throw new DataSourceOverSizeException("Table[" + ruleDataSource.getTableName() + "] is oversize with system config:[" + clusterInfo.getSkipDataSize() + "]");
                    }
                }
                partition.delete(0, partition.length());
            }
            if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
                continue;
            }
            if (!metaDataClient.fieldExist(ruleDataSource.getColName(), cols, mappingCol)) {
                throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "] {&RULE_DATASOURCE_BE_MOVED}");
            }
        } else if (StringUtils.isNotBlank(currentRule.getCsId())) {
            checkDatasourceInContextService(ruleDataSource, mappingCol, clusterName, userName, nodeName, currentRule.getCsId());
        }
    }
    if (CollectionUtils.isNotEmpty(datasourceSizeList) && currentRule.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())) {
        double left = datasourceSizeList.get(0);
        double right = datasourceSizeList.get(1);
        LOGGER.info("Current multi source rule left table size number[{}], right table size number[{}]", left, right);
        if (left == 0 && right == 0) {
            throw new BothNullDatasourceException("{&BOTH_SIDE_ARE_NULL}");
        } else if (left == 0) {
            throw new LeftNullDatasourceException("{&ONE_SIDE_ARE_NULL}");
        } else if (right == 0) {
            throw new RightNullDatasourceException("{&ONE_SIDE_ARE_NULL}");
        }
    }
}

Example 10 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method checkDatasourceInContextService.

private void checkDatasourceInContextService(RuleDataSource ruleDataSource, Map<String, String> mappingCol, String clusterName, String userName, String nodeName, String csId) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    GetUserTableByCsIdRequest getUserTableByCsIdRequest = new GetUserTableByCsIdRequest();
    getUserTableByCsIdRequest.setClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
    getUserTableByCsIdRequest.setLoginUser(userName);
    getUserTableByCsIdRequest.setCsId(csId);
    if (StringUtils.isBlank(nodeName)) {
        getUserTableByCsIdRequest.setNodeName(RuleConstraintEnum.DEFAULT_NODENAME.getValue());
    } else {
        getUserTableByCsIdRequest.setNodeName(nodeName);
    }
    DataInfo<CsTableInfoDetail> csTableInfoDetails = metaDataClient.getTableByCsId(getUserTableByCsIdRequest);
    if (csTableInfoDetails.getTotalCount() == 0) {
        LOGGER.info("Cannot find context service table with existed rules!");
        throw new UnExpectedRequestException("Table[" + ruleDataSource.getTableName() + "] {&RULE_DATASOURCE_BE_MOVED}");
    }
    for (CsTableInfoDetail csTableInfoDetail : csTableInfoDetails.getContent()) {
        if (csTableInfoDetail.getTableName().equals(ruleDataSource.getTableName())) {
            GetUserColumnByCsRequest getUserColumnByCsRequest = new GetUserColumnByCsRequest();
            getUserColumnByCsRequest.setClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
            getUserColumnByCsRequest.setContextKey(csTableInfoDetail.getContextKey());
            getUserColumnByCsRequest.setLoginUser(userName);
            getUserColumnByCsRequest.setCsId(csId);
            List<ColumnInfoDetail> cols = metaDataClient.getColumnByCsId(getUserColumnByCsRequest).getContent();
            if (ruleDataSource.getRule().getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
                continue;
            }
            if (!metaDataClient.fieldExist(ruleDataSource.getColName(), cols, mappingCol)) {
                throw new UnExpectedRequestException("Table[" + ruleDataSource.getTableName() + ":" + ruleDataSource.getColName() + "] {&RULE_DATASOURCE_BE_MOVED}");
            }
        }
    }
}