Examples with TableInfoDetail com.webank.wedatasphere.qualitis.metadata.response.table.TableInfoDetail used on opensource projects

Example 1 with TableInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.table.TableInfoDetail in project Qualitis by WeBankFinTech.

the class MetaDataServiceImpl method addMultiDbRules.

@Override
@Transactional(rollbackFor = { Exception.class, UnExpectedRequestException.class })
public String addMultiDbRules(MulDbRequest request) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    MulDbRequest.checkRequst(request);
    String loginUser = HttpUtils.getUserName(httpServletRequest);
    LOGGER.info("Start to get all tables with source database and target database.");
    List<String> sourceTableName = new ArrayList<>();
    List<String> targetTableName = new ArrayList<>();
    if (request.getSourceLinkisDataSourceId() == null) {
        GetUserTableByDbIdRequest sourceRequest = new GetUserTableByDbIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getSourceDb());
        sourceRequest.setProxyUser(request.getProxyUser());
        GeneralResponse<GetAllResponse<TableInfoDetail>> sourceTableInfos = getUserTableByDbId(sourceRequest);
        if (sourceTableInfos.getData().getTotal() <= 0) {
            throw new UnExpectedRequestException("Source database has no tables.");
        }
        sourceTableName.addAll(sourceTableInfos.getData().getData().stream().map(TableInfoDetail::getTableName).collect(Collectors.toList()));
    } else {
        Map response = getTablesByDataSource(request.getClusterName(), request.getProxyUser(), request.getSourceLinkisDataSourceId(), request.getSourceDb());
        List<String> tables = (List<String>) response.get("tables");
        for (String table : tables) {
            sourceTableName.add(table);
        }
    }
    if (request.getTargetLinkisDataSourceId() == null) {
        GetUserTableByDbIdRequest targetRequest = new GetUserTableByDbIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getTargetDb());
        targetRequest.setProxyUser(request.getProxyUser());
        GeneralResponse<GetAllResponse<TableInfoDetail>> targetTableInfos = getUserTableByDbId(targetRequest);
        if (targetTableInfos.getData().getTotal() <= 0) {
            throw new UnExpectedRequestException("Target database has no tables.");
        }
        targetTableName.addAll(targetTableInfos.getData().getData().stream().map(TableInfoDetail::getTableName).collect(Collectors.toList()));
    } else {
        Map response = getTablesByDataSource(request.getClusterName(), request.getProxyUser(), request.getTargetLinkisDataSourceId(), request.getTargetDb());
        List<String> tables = (List<String>) response.get("tables");
        for (String table : tables) {
            targetTableName.add(table);
        }
    }
    // Collect the table with the same name that exists in the source database and the target database.
    sourceTableName.retainAll(targetTableName);
    Set<String> ruleFailedTable = new HashSet<>();
    Long projectId = request.getProjectId();
    List<String> ruleTables = new ArrayList<>(ruleDataSourceDao.findByProjectId(projectId).stream().map(RuleDataSource::getTableName).distinct().collect(Collectors.toList()));
    // Filter tables for which rules have been created.
    sourceTableName = sourceTableName.stream().filter(tableName -> !ruleTables.contains(tableName)).collect(Collectors.toList());
    // Rule name index starts with the last created rule.
    int ruleIndex = ruleDao.findByProject(projectDao.findById(projectId)).size();
    // Filter tables with black list.
    sourceTableName = filterTablesWithBlackList(request.getBlackList(), sourceTableName);
    // White list.
    List<String> whiteList = request.getWhiteList();
    // Filter special table to compare with accuracy template.
    List<FilterRequest> filterRequests = request.getFilterRequests();
    List<FilterRequest> sameTableFilterRequests = filterRequests.stream().filter(filterRequest -> filterRequest.getSourceTable().equals(filterRequest.getTargetTable())).collect(Collectors.toList());
    List<String> filterSameTableName = sameTableFilterRequests.stream().map(FilterRequest::getSourceTable).collect(Collectors.toList());
    sourceTableName.removeAll(filterSameTableName);
    filterRequests.removeAll(sameTableFilterRequests);
    List<String> filterDiffTableName = filterRequests.stream().map(filterRequest -> filterRequest.getSourceTable() + SpecCharEnum.COLON.getValue() + filterRequest.getTargetTable()).collect(Collectors.toList());
    whiteList.removeAll(filterDiffTableName);
    filterRequests.addAll(sameTableFilterRequests);
    // Check flag: create rules in a loop, and only do permission verification for the first time.
    boolean check = true;
    int sourceTableSize = sourceTableName.size();
    int filterRequestsSize = filterRequests.size();
    int total = ruleIndex + sourceTableSize + whiteList.size() + filterRequestsSize;
    int i = ruleIndex;
    for (; i < ruleIndex + sourceTableSize; i++) {
        String currentSameTable = sourceTableName.get(i - ruleIndex);
        try {
            AddMultiSourceRuleRequest addMultiSourceRuleRequest = constructRequest(request, null, currentSameTable, currentSameTable, i, total, loginUser);
            LOGGER.info("Start to add {}th multi source rule.", i);
            if (i > 0) {
                check = false;
            }
            multiSourceRuleService.addMultiSourceRule(addMultiSourceRuleRequest, check);
        } catch (Exception e) {
            LOGGER.error("One of rule failed to add, rule index:[{}], table name: [{}], exception:{}", i, currentSameTable, e.getMessage());
            ruleFailedTable.add(currentSameTable);
        }
        LOGGER.info("Finish to add {}th multi source rule.", i);
    }
    // With white list.
    withWhiteListAndFilterRequest(i, ruleIndex, sourceTableSize, total, whiteList, filterRequests, request, loginUser, check);
    // Record the table that failed to create a rule.
    saveRuleFailedTableInLabel(ruleFailedTable, projectId);
    // projectEventService.record(projectId, loginUser, "create multi-source rule with dbs", "Success: " + (total - ruleFailedTable.size()) + "; Failed: " + ruleFailedTable.size(), EventTypeEnum.MODIFY_PROJECT.getCode());
    return request.getSourceDb() + " vs " + request.getSourceDb() + ": " + "{&CONTINUE_CREATE_RULES}";
}

Example 2 with TableInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.table.TableInfoDetail in project Qualitis by WeBankFinTech.

the class MetaDataController method getTablesByDataSource.

@GET
@Path("data_source/tables")
@Produces(MediaType.APPLICATION_JSON)
@Consumes(MediaType.APPLICATION_JSON)
public GeneralResponse<GetAllResponse<TableInfoDetail>> getTablesByDataSource(@QueryParam("clusterName") String clusterName, @QueryParam("proxyUser") String proxyUser, @QueryParam("dataSourceId") Long dataSourceId, @QueryParam("dbName") String dbName) {
    try {
        if (dataSourceId == null || StringUtils.isBlank(dbName) || StringUtils.isBlank(clusterName)) {
            throw new UnExpectedRequestException("Request " + "{&CAN_NOT_BE_NULL_OR_EMPTY}");
        }
        Map response = metaDataService.getTablesByDataSource(clusterName, proxyUser, dataSourceId, dbName);
        GetAllResponse allResponse = new GetAllResponse();
        List<String> tables = (List<String>) response.get("tables");
        List<TableInfoDetail> tableInfoDetails = new ArrayList<>(CollectionUtils.isEmpty(tables) ? 0 : tables.size());
        for (String table : tables) {
            TableInfoDetail tableInfoDetail = new TableInfoDetail(table);
            tableInfoDetails.add(tableInfoDetail);
        }
        allResponse.setTotal(CollectionUtils.isEmpty(tableInfoDetails) ? 0 : tableInfoDetails.size());
        allResponse.setData(tableInfoDetails);
        return new GeneralResponse<>("200", "{&GET_TABLE_SUCCESSFULLY}", allResponse);
    } catch (MetaDataAcquireFailedException e) {
        LOGGER.error(e.getMessage(), e);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (UnExpectedRequestException e) {
        LOGGER.error(e.getMessage(), e);
        return new GeneralResponse<>("500", e.getMessage(), null);
    } catch (Exception e) {
        LOGGER.error(e.getMessage(), e);
        return new GeneralResponse<>("500", "{&FAILED_TO_GET_TABLE_BY_DATABASE}", null);
    }
}

Example 3 with TableInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.table.TableInfoDetail in project Qualitis by WeBankFinTech.

the class MetaDataServiceImpl method getUserTableByDbId.

@Override
public GeneralResponse<GetAllResponse<TableInfoDetail>> getUserTableByDbId(GetUserTableByDbIdRequest request) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    // Check Arguments
    checkRequest(request);
    // Get login user
    String userName = HttpUtils.getUserName(httpServletRequest);
    if (StringUtils.isNotBlank(request.getProxyUser())) {
        userName = request.getProxyUser();
    }
    GetTableByUserAndDbRequest getTableByUserAndDbRequest = new GetTableByUserAndDbRequest(userName, request.getStartIndex(), request.getPageSize(), request.getClusterName(), request.getDbName());
    DataInfo<TableInfoDetail> response = metaDataClient.getTableByUserAndDb(getTableByUserAndDbRequest);
    GetAllResponse<TableInfoDetail> result = new GetAllResponse<>();
    result.setTotal(response.getTotalCount());
    result.setData(response.getContent());
    LOGGER.info("Succeed to get table by database. database: {}.{}", request.getClusterName(), request.getDbName());
    return new GeneralResponse<>("200", "{&GET_TABLE_SUCCESSFULLY}", result);
}

Example 4 with TableInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.table.TableInfoDetail in project Qualitis by WeBankFinTech.

the class MetaDataClientImpl method getTableByUserAndDb.

@Override
public DataInfo<TableInfoDetail> getTableByUserAndDb(GetTableByUserAndDbRequest request) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    ClusterInfo clusterInfo = checkClusterNameExists(request.getClusterName());
    String authUser = request.getLoginUser();
    // send request to get dbs
    String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getTablePath()).queryParam("database", request.getDbName()).toString();
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", authUser);
    headers.add("Token-Code", clusterInfo.getLinkisToken());
    HttpEntity<Object> entity = new HttpEntity<>(headers);
    LOGGER.info("Start to get table by user and cluster and db by linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
    Map<String, Object> response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
    LOGGER.info("Finished to get table by user and cluster and db by linkis. response: {}", response);
    if (!checkResponse(response)) {
        String message = (String) response.get("message");
        LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
        throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
    }
    List<String> allTables = ((List<Map<String, String>>) ((Map<String, Object>) response.get("data")).get("tables")).stream().map(o -> o.get("tableName")).collect(Collectors.toList());
    DataInfo<TableInfoDetail> dataInfo = new DataInfo<>(allTables.size());
    if (CollectionUtils.isEmpty(allTables)) {
        return dataInfo;
    }
    List<TableInfoDetail> details = new ArrayList<>();
    for (String data : allTables) {
        TableInfoDetail detail = new TableInfoDetail(data);
        details.add(detail);
    }
    dataInfo.setContent(details);
    return dataInfo;
}