Examples with ColumnInfoDetail com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail used on opensource projects

Example 11 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class RuleDataSourceServiceImpl method saveCustomColumn.

private List<DataSourceColumnRequest> saveCustomColumn(String clusterName, String db, String table, String loginUser, String csId, String nodeName, String funcContent, boolean fps, boolean cs) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>();
    List<ColumnInfoDetail> cols = new ArrayList<>();
    if (cs) {
        GetUserTableByCsIdRequest getUserTableByCsIdRequest = new GetUserTableByCsIdRequest();
        getUserTableByCsIdRequest.setClusterName(clusterName);
        getUserTableByCsIdRequest.setLoginUser(loginUser);
        getUserTableByCsIdRequest.setCsId(csId);
        getUserTableByCsIdRequest.setNodeName(nodeName);
        DataInfo<CsTableInfoDetail> csTableInfoDetails = metaDataClient.getTableByCsId(getUserTableByCsIdRequest);
        if (csTableInfoDetails.getTotalCount() == 0 && !fps) {
            LOGGER.info("Cannot find context service table with existed rules!");
            throw new UnExpectedRequestException("Table in sql {&DOES_NOT_EXIST}");
        }
        for (CsTableInfoDetail csTableInfoDetail : csTableInfoDetails.getContent()) {
            if (csTableInfoDetail.getTableName().equals(table)) {
                GetUserColumnByCsRequest getUserColumnByCsRequest = new GetUserColumnByCsRequest();
                getUserColumnByCsRequest.setClusterName(clusterName);
                getUserColumnByCsRequest.setContextKey(csTableInfoDetail.getContextKey());
                getUserColumnByCsRequest.setCsId(csId);
                getUserColumnByCsRequest.setLoginUser(loginUser);
                cols = metaDataClient.getColumnByCsId(getUserColumnByCsRequest).getContent();
                if (CollectionUtils.isEmpty(cols) && !fps) {
                    throw new UnExpectedRequestException("Table in sql {&DOES_NOT_EXIST}");
                }
            }
        }
    } else {
        cols = metaDataClient.getColumnInfo(clusterName, db, table, loginUser);
        if (CollectionUtils.isEmpty(cols) && !fps) {
            throw new UnExpectedRequestException("Table in sql {&DOES_NOT_EXIST}");
        }
    }
    // Save column info in use.
    for (ColumnInfoDetail detail : cols) {
        if (funcContent != null && funcContent.contains(detail.getFieldName())) {
            dataSourceColumnRequests.add(new DataSourceColumnRequest(detail.getFieldName(), detail.getDataType()));
        }
    }
    return dataSourceColumnRequests;
}

Example 12 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class MetaDataClientImpl method getColumnByUserAndTable.

@Override
public DataInfo<ColumnInfoDetail> getColumnByUserAndTable(GetColumnByUserAndTableRequest request) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    ClusterInfo clusterInfo = checkClusterNameExists(request.getClusterName());
    String authUser = request.getLoginUser();
    // send request to get dbs
    String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getColumnPath()).queryParam("database", request.getDbName()).queryParam("table", request.getTableName()).toString();
    HttpHeaders headers = new HttpHeaders();
    headers.setContentType(MediaType.APPLICATION_JSON);
    headers.add("Token-User", authUser);
    headers.add("Token-Code", clusterInfo.getLinkisToken());
    HttpEntity<Object> entity = new HttpEntity<>(headers);
    LOGGER.info("Start to get column by user and cluster and db and table by linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
    Map<String, Object> response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
    LOGGER.info("Finished to get table by user and cluster and and table by linkis. response: {}", response);
    if (!checkResponse(response)) {
        String message = (String) response.get("message");
        LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
        throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
    }
    List<Map<String, String>> allTables = ((List<Map<String, String>>) ((Map<String, Object>) response.get("data")).get("columns"));
    DataInfo<ColumnInfoDetail> dataInfo = new DataInfo<>(allTables.size());
    if (CollectionUtils.isEmpty(allTables)) {
        return dataInfo;
    }
    List<ColumnInfoDetail> details = new ArrayList<>();
    for (Map<String, String> table : allTables) {
        ColumnInfoDetail detail = new ColumnInfoDetail(table.get("columnName"), table.get("columnType"));
        details.add(detail);
    }
    dataInfo.setContent(details);
    return dataInfo;
}

Example 13 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class MetaDataClientImpl method fieldExist.

@Override
public boolean fieldExist(String col, List<ColumnInfoDetail> cols, Map<String, String> mappingCols) {
    // single or custom
    if (StringUtils.isNotBlank(col)) {
        // single field
        if (cols == null) {
            return false;
        }
        if (col.equals(SpecCharEnum.STAR.getValue())) {
            return cols != null && cols.size() > 0;
        }
        String[] colsInfo = col.split("\\|");
        int diff = colsInfo.length;
        for (String column : colsInfo) {
            for (ColumnInfoDetail columnInfoDetail : cols) {
                String realNameWithType = columnInfoDetail.getFieldName() + ":" + columnInfoDetail.getDataType();
                if (realNameWithType.equals(column)) {
                    diff--;
                    break;
                }
            }
        }
        return diff == 0;
    } else {
        // table level check or multi
        if (mappingCols != null && mappingCols.size() > 0) {
            int diff = mappingCols.size();
            for (String colName : mappingCols.keySet()) {
                for (ColumnInfoDetail columnInfoDetail : cols) {
                    if (columnInfoDetail.getFieldName().equals(colName) && columnInfoDetail.getDataType().equals(mappingCols.get(colName))) {
                        diff--;
                        break;
                    }
                }
            }
            return diff == 0;
        }
        return CollectionUtils.isNotEmpty(cols);
    }
}

Example 14 with ColumnInfoDetail

use of com.webank.wedatasphere.qualitis.metadata.response.column.ColumnInfoDetail in project Qualitis by WeBankFinTech.

the class MetaDataClientImpl method getColumnByCsId.

@Override
public DataInfo<ColumnInfoDetail> getColumnByCsId(GetUserColumnByCsRequest request) throws MetaDataAcquireFailedException, UnExpectedRequestException {
    DataInfo<ColumnInfoDetail> result = new DataInfo<>();
    List<ColumnInfoDetail> list = new ArrayList<>();
    try {
        LOGGER.info("Start to get columns with context service ID and table's context key. csId: {}, contextKey: {}", request.getCsId(), request.getContextKey());
        ClusterInfo clusterInfo = checkClusterNameExists(request.getClusterName());
        String authUser = request.getLoginUser();
        // send request
        String url;
        if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
            url = getPath(clusterInfo.getLinkisAddress()).path(QUERY_WORKFLOW_COLUMN_PATH).toString();
        } else {
            url = getPath(clusterInfo.getLinkisAddress()).path(QUERY_CS_COLUMN_PATH).toString();
        }
        HttpHeaders headers = new HttpHeaders();
        headers.setContentType(MediaType.APPLICATION_JSON);
        headers.add("Token-User", authUser);
        headers.add("Token-Code", clusterInfo.getLinkisToken());
        JSONObject jsonObject = new JSONObject();
        try {
            jsonObject.put("contextID", request.getCsId());
            jsonObject.put("contextKey", request.getContextKey());
        } catch (JSONException e) {
            LOGGER.error("Failed to construct http body json, exception is : {}", e);
        }
        HttpEntity<Object> entity = new HttpEntity<>(jsonObject.toString(), headers);
        LOGGER.info("Start to get column with context service ID and table's context key by restful API. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.POST, entity);
        Map<String, Object> response = restTemplate.exchange(url, HttpMethod.POST, entity, Map.class).getBody();
        if (!checkResponse(response)) {
            String message = (String) response.get("message");
            LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
            throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
        }
        LOGGER.info("Finished to get column with context service ID and table's context key by restful API. response: {}", response);
        Map<String, Object> data = (Map<String, Object>) response.get("data");
        List<Map<String, Object>> columns = (List<Map<String, Object>>) data.get("columns");
        if (columns == null || columns.size() == 0) {
            return result;
        }
        LOGGER.info("Successfully to get columns with context service ID and table's context key by restful API. csId: {}, contextKey: {}", request.getCsId(), request.getContextKey());
        for (Map<String, Object> column : columns) {
            ColumnInfoDetail columnInfoDetail = new ColumnInfoDetail();
            columnInfoDetail.setFieldName(column.get("columnName").toString());
            columnInfoDetail.setDataType(column.get("columnType").toString());
            columnInfoDetail.setColumnComment(column.get("columnComment") == null ? "" : column.get("columnComment").toString());
            columnInfoDetail.setPartitionField((Boolean) column.get("partitioned"));
            list.add(columnInfoDetail);
        }
        result.setTotalCount(columns.size());
        result.setContent(list);
    } catch (RestClientException e) {
        LOGGER.error(e.getMessage(), e);
        throw new MetaDataAcquireFailedException("Error! Can not get column by context service ID", 500);
    }
    return result;
}