Examples with ColumnInfoDetail com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail used on opensource projects

Example 1 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class RuleQueryServiceImpl method setRuleCount.

private void setRuleCount(List<ColumnInfoDetail> result, Long userId, String clusterName, String dbName, String tableName) {
    for (ColumnInfoDetail columnInfoDetail : result) {
        Integer count = ruleDataSourceCountDao.findCount(clusterName + "-" + dbName + "-" + tableName + "-" + columnInfoDetail.getFieldName() + ":" + columnInfoDetail.getDataType(), userId);
        if (count == null || count < 0) {
            LOGGER.error("Rule count of datasource is unlegal.");
        }
        columnInfoDetail.setRuleCount(count);
    }
}

Example 2 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class AddRuleRequestBuilder method solveDatasource.

private void solveDatasource(String datasource) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    List<DataSourceRequest> dataSourceRequests = new ArrayList<>(1);
    List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>(1);
    DataSourceRequest dataSourceRequest = new DataSourceRequest();
    String clusterName;
    String database;
    String table;
    String col;
    String filter;
    String[] datasourceStrs = datasource.split(SpecCharEnum.COLON.getValue());
    if (datasourceStrs.length > TWO) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    String dbAndTable = datasourceStrs[0];
    String dataSourceId = "";
    Matcher matcherId = DATA_SOURCE_ID.matcher(dbAndTable.toUpperCase());
    String dataSourceName = "";
    Matcher matcherName = DATA_SOURCE_NAME.matcher(dbAndTable.toUpperCase());
    while (matcherId.find()) {
        String group = matcherId.group();
        dataSourceId = group.replace(".(", "").replace(")", "").split("=")[1];
        int startIndex = dbAndTable.toUpperCase().indexOf(group);
        String replaceStr = dbAndTable.substring(startIndex, startIndex + group.length());
        dbAndTable = dbAndTable.replace(replaceStr, "");
    }
    if (StringUtils.isBlank(dataSourceId)) {
        while (matcherName.find()) {
            String group = matcherName.group();
            int startIndex = dbAndTable.toUpperCase().indexOf(group);
            String replaceStr = dbAndTable.substring(startIndex, startIndex + group.length());
            dataSourceName = replaceStr.replace(".(", "").replace(")", "").split("=")[1];
            dbAndTable = dbAndTable.replace(replaceStr, "");
        }
    }
    String[] datasources = dbAndTable.split(SpecCharEnum.PERIOD.getValue());
    filter = datasourceStrs[1];
    if (datasources.length != FOUR && datasources.length != THREE) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    clusterName = datasources[0];
    database = datasources[1];
    table = datasources[2];
    col = datasources.length >= FOUR ? datasources[3] : "";
    List<ColumnInfoDetail> cols = metaDataClient.getColumnInfo(clusterName, database, table, userName);
    if (StringUtils.isNotBlank(dataSourceId)) {
        LOGGER.info("Find data source connect. Data source ID: " + dataSourceId);
        dataSourceRequest.setLinkisDataSourceId(Long.parseLong(dataSourceId));
        GeneralResponse<Map> response = metaDataClient.getDataSourceInfoDetail(clusterName, userName, dataSourceRequest.getLinkisDataSourceId(), null);
        cols = metaDataClient.getColumnsByDataSource(clusterName, userName, dataSourceRequest.getLinkisDataSourceId(), database, table).getContent();
        Map dataSourceInfo = ((Map) response.getData().get("info"));
        String dataSourceInfoName = (String) dataSourceInfo.get("dataSourceName");
        String dataSourceInfoType = (String) ((Map) dataSourceInfo.get("dataSourceType")).get("name");
        dataSourceRequest.setLinkisDataSourceName(dataSourceInfoName);
        dataSourceRequest.setLinkisDataSourceType(dataSourceInfoType);
    } else if (StringUtils.isNotBlank(dataSourceName)) {
        LOGGER.info("Find data source connect. Data source name: " + dataSourceName);
        GeneralResponse<Map> response = metaDataClient.getDataSourceInfoDetailByName(clusterName, userName, dataSourceName);
        Map dataSourceInfo = ((Map) response.getData().get("info"));
        String dataSourceInfoName = (String) dataSourceInfo.get("dataSourceName");
        String dataSourceInfoType = (String) ((Map) dataSourceInfo.get("dataSourceType")).get("name");
        Integer currentDataSourceId = (Integer) dataSourceInfo.get("id");
        dataSourceRequest.setLinkisDataSourceId(currentDataSourceId.longValue());
        cols = metaDataClient.getColumnsByDataSource(clusterName, userName, currentDataSourceId.longValue(), database, table).getContent();
        dataSourceRequest.setLinkisDataSourceName(dataSourceInfoName);
        dataSourceRequest.setLinkisDataSourceType(dataSourceInfoType);
    }
    // For one or more fields
    if (StringUtils.isBlank(col)) {
        LOGGER.info("Table count check.");
    } else {
        boolean blackList = false;
        List<String> colsInDatasource = new ArrayList<>();
        if (col.startsWith(SpecCharEnum.MINUS.getValue())) {
            blackList = true;
            col = col.replace(SpecCharEnum.MINUS.getValue(), "");
        }
        if (col.contains(SpecCharEnum.LEFT_BRACKET.getValue()) && col.contains(SpecCharEnum.RIGHT_BRACKET.getValue())) {
            col = col.replace(SpecCharEnum.LEFT_BRACKET.getValue(), "").replace(SpecCharEnum.RIGHT_BRACKET.getValue(), "");
            for (String currentCol : col.split(SpecCharEnum.COMMA.getValue())) {
                colsInDatasource.add(currentCol);
            }
        } else {
            colsInDatasource.add(col);
        }
        if (blackList) {
            for (ColumnInfoDetail columnInfoDetail : cols) {
                if (colsInDatasource.contains(columnInfoDetail.getFieldName())) {
                    continue;
                } else {
                    DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest(columnInfoDetail.getFieldName(), columnInfoDetail.getDataType());
                    dataSourceColumnRequests.add(dataSourceColumnRequest);
                }
            }
        } else {
            for (String colName : colsInDatasource) {
                String type = cols.stream().filter(field -> field.getFieldName().equals(colName)).map(ColumnInfoDetail::getDataType).collect(Collectors.joining());
                DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest(colName, type);
                dataSourceColumnRequests.add(dataSourceColumnRequest);
            }
        }
        dataSourceRequest.setBlackList(blackList);
    }
    dataSourceRequest.setClusterName(clusterName);
    dataSourceRequest.setDbName(database);
    dataSourceRequest.setTableName(table);
    dataSourceRequest.setColNames(dataSourceColumnRequests);
    dataSourceRequest.setFilter(filter);
    dataSourceRequest.setProxyUser(userName);
    dataSourceRequests.add(dataSourceRequest);
    this.addRuleRequest.setDatasource(dataSourceRequests);
}

Example 3 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class MetaDataClientImpl method getColumnInfo.

@Override
public List<ColumnInfoDetail> getColumnInfo(String clusterName, String dbName, String tableName, String userName) throws MetaDataAcquireFailedException, UnExpectedRequestException {
    ClusterInfo clusterInfo = checkClusterNameExists(clusterName);
    // send request to get table comment.
    String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getColumnInfo()).queryParam("database", dbName).queryParam("tableName", tableName).toString();
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", userName);
    headers.add("Token-Code", clusterInfo.getLinkisToken());
    HttpEntity<Object> entity = new HttpEntity<>(headers);
    LOGGER.info("Start to get column info by user and cluster and db and table by linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
    Map<String, Object> response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
    LOGGER.info("Finished to get column info by user and cluster and db and table by linkis. response: {}", response);
    if (!checkResponse(response)) {
        String message = (String) response.get("message");
        LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
        throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
    }
    List<Map<String, Object>> tableFieldInfo = (List<Map<String, Object>>) ((Map<String, Object>) response.get("data")).get("tableFieldsInfo");
    List<ColumnInfoDetail> result = new ArrayList<>();
    for (Map<String, Object> map : tableFieldInfo) {
        ColumnInfoDetail columnInfoDetail = new ColumnInfoDetail();
        columnInfoDetail.setFieldName(map.get("name").toString());
        columnInfoDetail.setDataType(map.get("type").toString());
        if (map.get("length") != null && !"".equals(map.get("length").toString())) {
            columnInfoDetail.setColumnLen(Integer.parseInt(map.get("length").toString()));
        }
        columnInfoDetail.setColumnAlias(map.get("alias") == null ? "" : map.get("alias").toString());
        columnInfoDetail.setColumnComment(map.get("comment") == null ? "" : map.get("comment").toString());
        columnInfoDetail.setPrimary((Boolean) map.get("primary"));
        columnInfoDetail.setPartitionField((Boolean) map.get("partitionField"));
        result.add(columnInfoDetail);
    }
    return result;
}

Example 4 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class MetaDataClientImpl method getColumnsByDataSource.

@Override
public DataInfo<ColumnInfoDetail> getColumnsByDataSource(String clusterName, String authUser, Long dataSourceId, String dbName, String tableName) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    // Check existence of cluster name
    ClusterInfo clusterInfo = checkClusterNameExists(clusterName);
    // send request to get dbs
    String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getDatasourceColumn()).queryParam("system", "Qualitis").toString().replace("{DATA_SOURCE_ID}", dataSourceId.toString()).replace("{DATA_SOURCE_DB}", dbName).replace("{DATA_SOURCE_TABLE}", tableName);
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", authUser);
    headers.add("Token-Code", clusterInfo.getLinkisToken());
    HttpEntity<Object> entity = new HttpEntity<>(headers);
    LOGGER.info("Start to get columns by data source. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
    Map<String, Object> response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
    LOGGER.info("Finish to get columns by data source. response: {}", response);
    if (!checkResponse(response)) {
        String message = (String) response.get("message");
        LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
        throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
    }
    Map data = (Map) response.get(LinkisResponseKeyEnum.DATA.getKey());
    DataInfo<ColumnInfoDetail> result = new DataInfo<>();
    List<Map> tables = (List<Map>) data.get("columns");
    if (CollectionUtils.isEmpty(tables)) {
        LOGGER.info("No columns with data source to be choosed.");
    } else {
        List<ColumnInfoDetail> columnInfoDetailList = new ArrayList<>(tables.size());
        for (Map map : tables) {
            ColumnInfoDetail columnInfoDetail = new ColumnInfoDetail();
            columnInfoDetail.setFieldName((String) map.get("name"));
            columnInfoDetail.setDataType((String) map.get("type"));
            columnInfoDetailList.add(columnInfoDetail);
        }
        result.setTotalCount(columnInfoDetailList.size());
        result.setContent(columnInfoDetailList);
    }
    return result;
}

Example 5 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class MetaDataServiceImpl method constructRequest.

private AddMultiSourceRuleRequest constructRequest(MulDbRequest request, FilterRequest filterRequest, String currentSourceTable, String currentTargetTable, int index, int total, String loginUser) throws UnExpectedRequestException, MetaDataAcquireFailedException, IOException {
    LOGGER.info("Start to construct add multi source rule request.");
    // rule basic info.
    AddMultiSourceRuleRequest addMultiSourceRuleRequest = new AddMultiSourceRuleRequest();
    basicInfo(addMultiSourceRuleRequest, request, index, total);
    MultiDataSourceConfigRequest sourceConfigRequest = new MultiDataSourceConfigRequest();
    sourceConfigRequest.setLinkisDataSourceName(request.getSourceLinkisDataSourceName());
    sourceConfigRequest.setLinkisDataSourceType(request.getSourceLinkisDataSourceType());
    sourceConfigRequest.setLinkisDataSourceId(request.getSourceLinkisDataSourceId());
    sourceConfigRequest.setProxyUser(request.getProxyUser());
    sourceConfigRequest.setDbName(request.getSourceDb());
    sourceConfigRequest.setContextService(false);
    MultiDataSourceConfigRequest targetConfigRequest = new MultiDataSourceConfigRequest();
    targetConfigRequest.setLinkisDataSourceName(request.getTargetLinkisDataSourceName());
    targetConfigRequest.setLinkisDataSourceType(request.getTargetLinkisDataSourceType());
    targetConfigRequest.setLinkisDataSourceId(request.getTargetLinkisDataSourceId());
    targetConfigRequest.setProxyUser(request.getProxyUser());
    targetConfigRequest.setDbName(request.getTargetDb());
    targetConfigRequest.setContextService(false);
    if (filterRequest == null) {
        // source table and target table
        sourceConfigRequest.setTableName(currentSourceTable);
        sourceConfigRequest.setFilter("true");
        targetConfigRequest.setTableName(currentTargetTable);
        targetConfigRequest.setFilter("true");
        addMultiSourceRuleRequest.setMultiSourceRuleTemplateId(20L);
        List<AlarmConfigRequest> alarmConfigRequests = new ArrayList<>();
        alarmConfigRequests.add(new AlarmConfigRequest(645L, CheckTemplateEnum.FIXED_VALUE.getCode(), CompareTypeEnum.EQUAL.getCode(), 0.0));
        addMultiSourceRuleRequest.setAlarmVariable(alarmConfigRequests);
    } else {
        sourceConfigRequest.setTableName(filterRequest.getSourceTable());
        if (StringUtils.isNotBlank(filterRequest.getSourceFilter())) {
            sourceConfigRequest.setFilter("! (" + filterRequest.getSourceFilter() + ")");
        } else {
            sourceConfigRequest.setFilter("true");
        }
        targetConfigRequest.setTableName(filterRequest.getTargetTable());
        if (StringUtils.isNotBlank(filterRequest.getTargetFilter())) {
            targetConfigRequest.setFilter("! (" + filterRequest.getTargetFilter() + ")");
        } else {
            targetConfigRequest.setFilter("true");
        }
        addMultiSourceRuleRequest.setMultiSourceRuleTemplateId(17L);
        List<AlarmConfigRequest> alarmConfigRequests = new ArrayList<>();
        alarmConfigRequests.add(new AlarmConfigRequest(33L, CheckTemplateEnum.FIXED_VALUE.getCode(), CompareTypeEnum.EQUAL.getCode(), 0.0));
        addMultiSourceRuleRequest.setAlarmVariable(alarmConfigRequests);
    }
    addMultiSourceRuleRequest.setSource(sourceConfigRequest);
    addMultiSourceRuleRequest.setTarget(targetConfigRequest);
    // All fields mappings
    LOGGER.info("Start to get all fields with source database's table and target database's table.");
    List<ColumnInfoDetail> sourceFields;
    List<ColumnInfoDetail> targetFields;
    if (request.getSourceLinkisDataSourceId() == null) {
        GetUserColumnByTableIdRequest getUserColumnSourceRequest = new GetUserColumnByTableIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getSourceDb(), filterRequest == null ? currentSourceTable : filterRequest.getSourceTable());
        getUserColumnSourceRequest.setProxyUser(request.getProxyUser());
        getUserColumnSourceRequest.setLoginUser(loginUser);
        sourceFields = getUserColumnByTableId(getUserColumnSourceRequest).getData().getData();
    } else {
        sourceFields = getColumnsByDataSource(request.getClusterName(), request.getProxyUser(), request.getSourceLinkisDataSourceId(), request.getSourceDb(), currentSourceTable).getData().getData();
    }
    if (request.getTargetLinkisDataSourceId() == null) {
        GetUserColumnByTableIdRequest getUserColumnTargetRequest = new GetUserColumnByTableIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getTargetDb(), filterRequest == null ? currentTargetTable : filterRequest.getTargetTable());
        getUserColumnTargetRequest.setProxyUser(request.getProxyUser());
        getUserColumnTargetRequest.setLoginUser(loginUser);
        targetFields = getUserColumnByTableId(getUserColumnTargetRequest).getData().getData();
    } else {
        targetFields = getColumnsByDataSource(request.getClusterName(), request.getProxyUser(), request.getTargetLinkisDataSourceId(), request.getTargetDb(), currentTargetTable).getData().getData();
    }
    if (CollectionUtils.isEmpty(sourceFields) || CollectionUtils.isEmpty(targetFields)) {
        throw new UnExpectedRequestException("There is table which has none field");
    }
    LOGGER.info("Success to get all fields with source database's table and target database's table.");
    // Sorted columns.
    Collections.sort(sourceFields, new Comparator<ColumnInfoDetail>() {

        @Override
        public int compare(ColumnInfoDetail front, ColumnInfoDetail back) {
            return front.getFieldName().compareTo(back.getFieldName());
        }
    });
    Collections.sort(targetFields, new Comparator<ColumnInfoDetail>() {

        @Override
        public int compare(ColumnInfoDetail front, ColumnInfoDetail back) {
            return front.getFieldName().compareTo(back.getFieldName());
        }
    });
    String sourceFieldStr = sourceFields.stream().map(ColumnInfoDetail::getFieldName).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
    String sourceFieldTypeStr = sourceFields.stream().map(ColumnInfoDetail::getDataType).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
    String targetFieldStr = targetFields.stream().map(ColumnInfoDetail::getFieldName).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
    String targetFieldTypeStr = targetFields.stream().map(ColumnInfoDetail::getDataType).collect(Collectors.joining(SpecCharEnum.COMMA.getValue()));
    if (sourceFields.size() == 0 || targetFields.size() == 0 || sourceFields.size() != targetFields.size() || !sourceFieldStr.equals(targetFieldStr) || !sourceFieldTypeStr.equals(targetFieldTypeStr)) {
        throw new UnExpectedRequestException("Create multi-db rules failed, because the fields' name of table is different.");
    }
    List<MultiDataSourceJoinConfigRequest> mappings = new ArrayList<>(sourceFields.size());
    for (int j = 0; j < sourceFields.size() && j < targetFields.size(); j++) {
        ColumnInfoDetail currentSourceField = sourceFields.get(j);
        ColumnInfoDetail currentTargetField = targetFields.get(j);
        if (filterRequest != null && CollectionUtils.isNotEmpty(filterRequest.getFilterColumnList()) && filterRequest.getFilterColumnList().contains(currentSourceField.getFieldName())) {
            continue;
        }
        if (currentSourceField.getFieldName().equals(currentTargetField.getFieldName()) && currentSourceField.getDataType().equals(currentTargetField.getDataType())) {
            // TODO: continue
            MultiDataSourceJoinConfigRequest multiDataSourceJoinConfigRequest = new MultiDataSourceJoinConfigRequest();
            MultiDataSourceJoinColumnRequest leftJoinColumnRequest = new MultiDataSourceJoinColumnRequest("tmp1.".concat(currentSourceField.getFieldName()), currentSourceField.getDataType());
            MultiDataSourceJoinColumnRequest rightJoinColumnRequest = new MultiDataSourceJoinColumnRequest("tmp2.".concat(currentTargetField.getFieldName()), currentTargetField.getDataType());
            multiDataSourceJoinConfigRequest.setOperation(MappingOperationEnum.EQUAL.getCode());
            multiDataSourceJoinConfigRequest.setLeft(Arrays.asList(leftJoinColumnRequest));
            multiDataSourceJoinConfigRequest.setRight(Arrays.asList(rightJoinColumnRequest));
            multiDataSourceJoinConfigRequest.setLeftStatement("tmp1.".concat(currentSourceField.getFieldName()));
            multiDataSourceJoinConfigRequest.setRightStatement("tmp2.".concat(currentTargetField.getFieldName()));
            mappings.add(multiDataSourceJoinConfigRequest);
        } else {
            throw new UnExpectedRequestException("Create multi-db rules failed, because the fields' name of tables is different");
        }
    }
    addMultiSourceRuleRequest.setMappings(mappings);
    addMultiSourceRuleRequest.setLoginUser(loginUser);
    LOGGER.info("Success to construct add multi source rule request. Request[{}]", new ObjectMapper().writeValueAsString(addMultiSourceRuleRequest));
    return addMultiSourceRuleRequest;
}