Search in sources :

Example 1 with FilterRequest

use of com.webank.wedatasphere.qualitis.request.FilterRequest in project Qualitis by WeBankFinTech.

the class MetaDataServiceImpl method addMultiDbRules.

@Override
@Transactional(rollbackFor = { Exception.class, UnExpectedRequestException.class })
public String addMultiDbRules(MulDbRequest request) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    MulDbRequest.checkRequst(request);
    String loginUser = HttpUtils.getUserName(httpServletRequest);
    LOGGER.info("Start to get all tables with source database and target database.");
    List<String> sourceTableName = new ArrayList<>();
    List<String> targetTableName = new ArrayList<>();
    if (request.getSourceLinkisDataSourceId() == null) {
        GetUserTableByDbIdRequest sourceRequest = new GetUserTableByDbIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getSourceDb());
        sourceRequest.setProxyUser(request.getProxyUser());
        GeneralResponse<GetAllResponse<TableInfoDetail>> sourceTableInfos = getUserTableByDbId(sourceRequest);
        if (sourceTableInfos.getData().getTotal() <= 0) {
            throw new UnExpectedRequestException("Source database has no tables.");
        }
        sourceTableName.addAll(sourceTableInfos.getData().getData().stream().map(TableInfoDetail::getTableName).collect(Collectors.toList()));
    } else {
        Map response = getTablesByDataSource(request.getClusterName(), request.getProxyUser(), request.getSourceLinkisDataSourceId(), request.getSourceDb());
        List<String> tables = (List<String>) response.get("tables");
        for (String table : tables) {
            sourceTableName.add(table);
        }
    }
    if (request.getTargetLinkisDataSourceId() == null) {
        GetUserTableByDbIdRequest targetRequest = new GetUserTableByDbIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getTargetDb());
        targetRequest.setProxyUser(request.getProxyUser());
        GeneralResponse<GetAllResponse<TableInfoDetail>> targetTableInfos = getUserTableByDbId(targetRequest);
        if (targetTableInfos.getData().getTotal() <= 0) {
            throw new UnExpectedRequestException("Target database has no tables.");
        }
        targetTableName.addAll(targetTableInfos.getData().getData().stream().map(TableInfoDetail::getTableName).collect(Collectors.toList()));
    } else {
        Map response = getTablesByDataSource(request.getClusterName(), request.getProxyUser(), request.getTargetLinkisDataSourceId(), request.getTargetDb());
        List<String> tables = (List<String>) response.get("tables");
        for (String table : tables) {
            targetTableName.add(table);
        }
    }
    // Collect the table with the same name that exists in the source database and the target database.
    sourceTableName.retainAll(targetTableName);
    Set<String> ruleFailedTable = new HashSet<>();
    Long projectId = request.getProjectId();
    List<String> ruleTables = new ArrayList<>(ruleDataSourceDao.findByProjectId(projectId).stream().map(RuleDataSource::getTableName).distinct().collect(Collectors.toList()));
    // Filter tables for which rules have been created.
    sourceTableName = sourceTableName.stream().filter(tableName -> !ruleTables.contains(tableName)).collect(Collectors.toList());
    // Rule name index starts with the last created rule.
    int ruleIndex = ruleDao.findByProject(projectDao.findById(projectId)).size();
    // Filter tables with black list.
    sourceTableName = filterTablesWithBlackList(request.getBlackList(), sourceTableName);
    // White list.
    List<String> whiteList = request.getWhiteList();
    // Filter special table to compare with accuracy template.
    List<FilterRequest> filterRequests = request.getFilterRequests();
    List<FilterRequest> sameTableFilterRequests = filterRequests.stream().filter(filterRequest -> filterRequest.getSourceTable().equals(filterRequest.getTargetTable())).collect(Collectors.toList());
    List<String> filterSameTableName = sameTableFilterRequests.stream().map(FilterRequest::getSourceTable).collect(Collectors.toList());
    sourceTableName.removeAll(filterSameTableName);
    filterRequests.removeAll(sameTableFilterRequests);
    List<String> filterDiffTableName = filterRequests.stream().map(filterRequest -> filterRequest.getSourceTable() + SpecCharEnum.COLON.getValue() + filterRequest.getTargetTable()).collect(Collectors.toList());
    whiteList.removeAll(filterDiffTableName);
    filterRequests.addAll(sameTableFilterRequests);
    // Check flag: create rules in a loop, and only do permission verification for the first time.
    boolean check = true;
    int sourceTableSize = sourceTableName.size();
    int filterRequestsSize = filterRequests.size();
    int total = ruleIndex + sourceTableSize + whiteList.size() + filterRequestsSize;
    int i = ruleIndex;
    for (; i < ruleIndex + sourceTableSize; i++) {
        String currentSameTable = sourceTableName.get(i - ruleIndex);
        try {
            AddMultiSourceRuleRequest addMultiSourceRuleRequest = constructRequest(request, null, currentSameTable, currentSameTable, i, total, loginUser);
            LOGGER.info("Start to add {}th multi source rule.", i);
            if (i > 0) {
                check = false;
            }
            multiSourceRuleService.addMultiSourceRule(addMultiSourceRuleRequest, check);
        } catch (Exception e) {
            LOGGER.error("One of rule failed to add, rule index:[{}], table name: [{}], exception:{}", i, currentSameTable, e.getMessage());
            ruleFailedTable.add(currentSameTable);
        }
        LOGGER.info("Finish to add {}th multi source rule.", i);
    }
    // With white list.
    withWhiteListAndFilterRequest(i, ruleIndex, sourceTableSize, total, whiteList, filterRequests, request, loginUser, check);
    // Record the table that failed to create a rule.
    saveRuleFailedTableInLabel(ruleFailedTable, projectId);
    // projectEventService.record(projectId, loginUser, "create multi-source rule with dbs", "Success: " + (total - ruleFailedTable.size()) + "; Failed: " + ruleFailedTable.size(), EventTypeEnum.MODIFY_PROJECT.getCode());
    return request.getSourceDb() + " vs " + request.getSourceDb() + ": " + "{&CONTINUE_CREATE_RULES}";
}
Also used : GetUserTableByDbIdRequest(com.webank.wedatasphere.qualitis.request.GetUserTableByDbIdRequest) Arrays(java.util.Arrays) StringUtils(org.apache.commons.lang.StringUtils) LoggerFactory(org.slf4j.LoggerFactory) GetUserColumnByCsRequest(com.webank.wedatasphere.qualitis.metadata.request.GetUserColumnByCsRequest) Autowired(org.springframework.beans.factory.annotation.Autowired) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) ClusterInfoDao(com.webank.wedatasphere.qualitis.dao.ClusterInfoDao) HttpUtils(com.webank.wedatasphere.qualitis.util.HttpUtils) Map(java.util.Map) MultiDataSourceConfigRequest(com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceConfigRequest) GetClusterByUserRequest(com.webank.wedatasphere.qualitis.metadata.request.GetClusterByUserRequest) ProjectService(com.webank.wedatasphere.qualitis.project.service.ProjectService) GetAllResponse(com.webank.wedatasphere.qualitis.response.GetAllResponse) AlarmConfigRequest(com.webank.wedatasphere.qualitis.rule.request.AlarmConfigRequest) ProjectEventService(com.webank.wedatasphere.qualitis.project.service.ProjectEventService) GetUserDbByClusterRequest(com.webank.wedatasphere.qualitis.request.GetUserDbByClusterRequest) GetAllClusterResponse(com.webank.wedatasphere.qualitis.response.GetAllClusterResponse) DataSourceModifyRequest(com.webank.wedatasphere.qualitis.request.DataSourceModifyRequest) AddMultiSourceRuleRequest(com.webank.wedatasphere.qualitis.rule.request.multi.AddMultiSourceRuleRequest) MappingOperationEnum(com.webank.wedatasphere.qualitis.rule.constant.MappingOperationEnum) Context(javax.ws.rs.core.Context) TableInfoDetail(com.webank.wedatasphere.qualitis.metadata.response.table.TableInfoDetail) Set(java.util.Set) Collectors(java.util.stream.Collectors) MetaDataAcquireFailedException(com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException) GetDbByUserAndClusterRequest(com.webank.wedatasphere.qualitis.metadata.request.GetDbByUserAndClusterRequest) List(java.util.List) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) MetaDataClient(com.webank.wedatasphere.qualitis.metadata.client.MetaDataClient) Pattern(java.util.regex.Pattern) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) DataSourceConnectRequest(com.webank.wedatasphere.qualitis.request.DataSourceConnectRequest) GetUserClusterRequest(com.webank.wedatasphere.qualitis.request.GetUserClusterRequest) UnsupportedEncodingException(java.io.UnsupportedEncodingException) BlackListFilterTypeEnum(com.webank.wedatasphere.qualitis.constant.BlackListFilterTypeEnum) ClusterInfoDetail(com.webank.wedatasphere.qualitis.metadata.response.cluster.ClusterInfoDetail) GetTableByUserAndDbRequest(com.webank.wedatasphere.qualitis.metadata.request.GetTableByUserAndDbRequest) Project(com.webank.wedatasphere.qualitis.project.entity.Project) RuleLimitationService(com.webank.wedatasphere.qualitis.rule.service.RuleLimitationService) SpecCharEnum(com.webank.wedatasphere.qualitis.constant.SpecCharEnum) MulDbRequest(com.webank.wedatasphere.qualitis.request.MulDbRequest) RuleDataSourceDao(com.webank.wedatasphere.qualitis.rule.dao.RuleDataSourceDao) ProjectDao(com.webank.wedatasphere.qualitis.project.dao.ProjectDao) DataSourceParamModifyRequest(com.webank.wedatasphere.qualitis.request.DataSourceParamModifyRequest) FilterRequest(com.webank.wedatasphere.qualitis.request.FilterRequest) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) HttpServletRequest(javax.servlet.http.HttpServletRequest) CheckTemplateEnum(com.webank.wedatasphere.qualitis.rule.constant.CheckTemplateEnum) CollectionUtils(org.apache.commons.collections.CollectionUtils) Service(org.springframework.stereotype.Service) GetUserTableByCsIdRequest(com.webank.wedatasphere.qualitis.metadata.request.GetUserTableByCsIdRequest) CompareTypeEnum(com.webank.wedatasphere.qualitis.rule.constant.CompareTypeEnum) MetaDataService(com.webank.wedatasphere.qualitis.service.MetaDataService) MultiDataSourceJoinColumnRequest(com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinColumnRequest) CsTableInfoDetail(com.webank.wedatasphere.qualitis.metadata.response.table.CsTableInfoDetail) Logger(org.slf4j.Logger) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException) GetColumnByUserAndTableRequest(com.webank.wedatasphere.qualitis.metadata.request.GetColumnByUserAndTableRequest) GetUserColumnByTableIdRequest(com.webank.wedatasphere.qualitis.request.GetUserColumnByTableIdRequest) ColumnInfoDetail(com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail) DbInfoDetail(com.webank.wedatasphere.qualitis.metadata.response.db.DbInfoDetail) DataInfo(com.webank.wedatasphere.qualitis.metadata.response.DataInfo) MultiSourceRuleService(com.webank.wedatasphere.qualitis.rule.service.MultiSourceRuleService) RuleDao(com.webank.wedatasphere.qualitis.rule.dao.RuleDao) MultiDataSourceJoinConfigRequest(com.webank.wedatasphere.qualitis.rule.request.multi.MultiDataSourceJoinConfigRequest) GetUserTableByDbIdRequest(com.webank.wedatasphere.qualitis.request.GetUserTableByDbIdRequest) Comparator(java.util.Comparator) Collections(java.util.Collections) Transactional(org.springframework.transaction.annotation.Transactional) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) ArrayList(java.util.ArrayList) MetaDataAcquireFailedException(com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException) FilterRequest(com.webank.wedatasphere.qualitis.request.FilterRequest) List(java.util.List) ArrayList(java.util.ArrayList) GetAllResponse(com.webank.wedatasphere.qualitis.response.GetAllResponse) TableInfoDetail(com.webank.wedatasphere.qualitis.metadata.response.table.TableInfoDetail) CsTableInfoDetail(com.webank.wedatasphere.qualitis.metadata.response.table.CsTableInfoDetail) Map(java.util.Map) HashSet(java.util.HashSet) AddMultiSourceRuleRequest(com.webank.wedatasphere.qualitis.rule.request.multi.AddMultiSourceRuleRequest) Transactional(org.springframework.transaction.annotation.Transactional)

Example 2 with FilterRequest

use of com.webank.wedatasphere.qualitis.request.FilterRequest in project Qualitis by WeBankFinTech.

the class MetaDataServiceImpl method withWhiteListAndFilterRequest.

private void withWhiteListAndFilterRequest(int i, int ruleIndex, int sourceTableSize, int total, List<String> whiteList, List<FilterRequest> filterRequests, MulDbRequest request, String loginUser, boolean check) {
    for (; i < total - filterRequests.size(); i++) {
        String currentTables = whiteList.get(i - ruleIndex - sourceTableSize);
        String currentSourceTable = currentTables.split(SpecCharEnum.COLON.getValue())[0];
        String currentTargetTable = currentTables.split(SpecCharEnum.COLON.getValue())[1];
        try {
            // Construct add multi source rule request.
            AddMultiSourceRuleRequest addMultiSourceRuleRequest = constructRequest(request, null, currentSourceTable, currentTargetTable, i, total, loginUser);
            LOGGER.info("Start to add {}th multi source rule.", i);
            if (ruleIndex > 0) {
                check = false;
            }
            multiSourceRuleService.addMultiSourceRule(addMultiSourceRuleRequest, check);
        } catch (Exception e) {
            LOGGER.error("One of rule failed to add, rule index:[{}], table name: [{}], exception:{}", i, currentSourceTable + " VS " + currentTargetTable, e.getMessage());
        }
        LOGGER.info("Finish to add {}th multi source rule.", i);
    }
    for (; i < total; i++) {
        FilterRequest filterRequest = filterRequests.get(i - ruleIndex - sourceTableSize - whiteList.size());
        try {
            AddMultiSourceRuleRequest addMultiSourceRuleRequest = constructRequest(request, filterRequest, "", "", i, total, loginUser);
            LOGGER.info("Start to add {}th multi source rule.", i);
            if (i > 0) {
                check = false;
            }
            multiSourceRuleService.addMultiSourceRule(addMultiSourceRuleRequest, check);
        } catch (Exception e) {
            LOGGER.error("One of rule failed to add, rule index:[{}], table name: [{}], exception:{}", i, filterRequest.getSourceTable() + SpecCharEnum.COMMA.getValue() + filterRequest.getTargetTable(), e.getMessage());
        }
        LOGGER.info("Finish to add {}th multi source rule.", i);
    }
}
Also used : FilterRequest(com.webank.wedatasphere.qualitis.request.FilterRequest) MetaDataAcquireFailedException(com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) UnsupportedEncodingException(java.io.UnsupportedEncodingException) JsonProcessingException(com.fasterxml.jackson.core.JsonProcessingException) IOException(java.io.IOException) AddMultiSourceRuleRequest(com.webank.wedatasphere.qualitis.rule.request.multi.AddMultiSourceRuleRequest)

Aggregations

JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)2 UnExpectedRequestException (com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException)2 MetaDataAcquireFailedException (com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException)2 FilterRequest (com.webank.wedatasphere.qualitis.request.FilterRequest)2 AddMultiSourceRuleRequest (com.webank.wedatasphere.qualitis.rule.request.multi.AddMultiSourceRuleRequest)2 IOException (java.io.IOException)2 UnsupportedEncodingException (java.io.UnsupportedEncodingException)2 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 BlackListFilterTypeEnum (com.webank.wedatasphere.qualitis.constant.BlackListFilterTypeEnum)1 SpecCharEnum (com.webank.wedatasphere.qualitis.constant.SpecCharEnum)1 ClusterInfoDao (com.webank.wedatasphere.qualitis.dao.ClusterInfoDao)1 ClusterInfo (com.webank.wedatasphere.qualitis.entity.ClusterInfo)1 MetaDataClient (com.webank.wedatasphere.qualitis.metadata.client.MetaDataClient)1 GetClusterByUserRequest (com.webank.wedatasphere.qualitis.metadata.request.GetClusterByUserRequest)1 GetColumnByUserAndTableRequest (com.webank.wedatasphere.qualitis.metadata.request.GetColumnByUserAndTableRequest)1 GetDbByUserAndClusterRequest (com.webank.wedatasphere.qualitis.metadata.request.GetDbByUserAndClusterRequest)1 GetTableByUserAndDbRequest (com.webank.wedatasphere.qualitis.metadata.request.GetTableByUserAndDbRequest)1 GetUserColumnByCsRequest (com.webank.wedatasphere.qualitis.metadata.request.GetUserColumnByCsRequest)1 GetUserTableByCsIdRequest (com.webank.wedatasphere.qualitis.metadata.request.GetUserTableByCsIdRequest)1 DataInfo (com.webank.wedatasphere.qualitis.metadata.response.DataInfo)1