use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.
the class MetaDataClientImpl method getDbsByDataSource.
@Override
public Map getDbsByDataSource(String clusterName, String authUser, Long dataSourceId) throws UnExpectedRequestException, MetaDataAcquireFailedException {
// Check existence of cluster name
ClusterInfo clusterInfo = checkClusterNameExists(clusterName);
// send request to get dbs
String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getDatasourceDb()).queryParam("system", "Qualitis").toString().replace("{DATA_SOURCE_ID}", dataSourceId.toString());
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.add("Token-User", authUser);
headers.add("Token-Code", clusterInfo.getLinkisToken());
HttpEntity<Object> entity = new HttpEntity<>(headers);
LOGGER.info("Start to get dbs by data source. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
Map<String, Object> response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
LOGGER.info("Finish to get dbs by data source. response: {}", response);
if (!checkResponse(response)) {
String message = (String) response.get("message");
LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
}
Map data = (Map) response.get(LinkisResponseKeyEnum.DATA.getKey());
List<Map> dbs = (List<Map>) data.get("dbs");
if (CollectionUtils.isEmpty(dbs)) {
LOGGER.info("No dbs with data source to be choosed.");
}
return data;
}
use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.
the class MetaDataClientImpl method getColumnsByDataSource.
@Override
public DataInfo<ColumnInfoDetail> getColumnsByDataSource(String clusterName, String authUser, Long dataSourceId, String dbName, String tableName) throws UnExpectedRequestException, MetaDataAcquireFailedException {
// Check existence of cluster name
ClusterInfo clusterInfo = checkClusterNameExists(clusterName);
// send request to get dbs
String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getDatasourceColumn()).queryParam("system", "Qualitis").toString().replace("{DATA_SOURCE_ID}", dataSourceId.toString()).replace("{DATA_SOURCE_DB}", dbName).replace("{DATA_SOURCE_TABLE}", tableName);
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.add("Token-User", authUser);
headers.add("Token-Code", clusterInfo.getLinkisToken());
HttpEntity<Object> entity = new HttpEntity<>(headers);
LOGGER.info("Start to get columns by data source. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
Map<String, Object> response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
LOGGER.info("Finish to get columns by data source. response: {}", response);
if (!checkResponse(response)) {
String message = (String) response.get("message");
LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
}
Map data = (Map) response.get(LinkisResponseKeyEnum.DATA.getKey());
DataInfo<ColumnInfoDetail> result = new DataInfo<>();
List<Map> tables = (List<Map>) data.get("columns");
if (CollectionUtils.isEmpty(tables)) {
LOGGER.info("No columns with data source to be choosed.");
} else {
List<ColumnInfoDetail> columnInfoDetailList = new ArrayList<>(tables.size());
for (Map map : tables) {
ColumnInfoDetail columnInfoDetail = new ColumnInfoDetail();
columnInfoDetail.setFieldName((String) map.get("name"));
columnInfoDetail.setDataType((String) map.get("type"));
columnInfoDetailList.add(columnInfoDetail);
}
result.setTotalCount(columnInfoDetailList.size());
result.setContent(columnInfoDetailList);
}
return result;
}
use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.
the class MetaDataClientImpl method createDataSource.
@Override
public GeneralResponse<Map> createDataSource(String clusterName, String authUser, String jsonRequest) throws UnExpectedRequestException, MetaDataAcquireFailedException {
// Check existence of cluster name
ClusterInfo clusterInfo = checkClusterNameExists(clusterName);
// send request to get dbs
String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getDatasourceCreate()).toString();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.add("Token-User", authUser);
headers.add("Token-Code", clusterInfo.getLinkisToken());
HttpEntity<Object> entity = new HttpEntity<>(jsonRequest, headers);
LOGGER.info("Start to create data source by user and cluster by linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.POST, entity);
Map<String, Object> response = restTemplate.exchange(url, HttpMethod.POST, entity, Map.class).getBody();
LOGGER.info("Finish to create data source by user and cluster by linkis. response: {}", response);
if (!checkResponse(response)) {
String message = (String) response.get("message");
LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
}
Map data = (Map) response.get(LinkisResponseKeyEnum.DATA.getKey());
return new GeneralResponse<>("200", "Success to create datasource", data);
}
use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.
the class MetaDataClientImpl method getDataSourceInfoPage.
@Override
public GeneralResponse<Map> getDataSourceInfoPage(String clusterName, String authUser, int page, int size, String searchName, Long typeId) throws UnExpectedRequestException, MetaDataAcquireFailedException, UnsupportedEncodingException {
// Check existence of cluster name
ClusterInfo clusterInfo = checkClusterNameExists(clusterName);
// send request to get dbs
UriBuilder uriBuilder = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getDatasourceInfo()).queryParam("currentPage", page).queryParam("pageSize", size);
if (StringUtils.isNotBlank(searchName)) {
uriBuilder.queryParam("name", searchName);
}
if (typeId != null) {
uriBuilder.queryParam("typeId", typeId);
}
String url = uriBuilder.toString();
url = URLDecoder.decode(url, "UTF-8");
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.add("Token-User", authUser);
headers.add("Token-Code", clusterInfo.getLinkisToken());
HttpEntity<Object> entity = new HttpEntity<>(headers);
LOGGER.info("Start to get data source info by user and cluster by linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
Map<String, Object> response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
LOGGER.info("Finish to get data source info by user and cluster by linkis. response: {}", response);
if (!checkResponse(response)) {
String message = (String) response.get("message");
LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
throw new MetaDataAcquireFailedException("Error! Can not get meta data from linkis, exception: " + message);
}
Map data = (Map) response.get(LinkisResponseKeyEnum.DATA.getKey());
List<Map> types = (List<Map>) data.get("query_list");
return new GeneralResponse<>("200", "Success to get datasource info", data);
}
use of com.webank.wedatasphere.qualitis.metadata.exception.MetaDataAcquireFailedException in project Qualitis by WeBankFinTech.
the class MetaDataServiceImpl method addMultiDbRules.
@Override
@Transactional(rollbackFor = { Exception.class, UnExpectedRequestException.class })
public String addMultiDbRules(MulDbRequest request) throws UnExpectedRequestException, MetaDataAcquireFailedException {
MulDbRequest.checkRequst(request);
String loginUser = HttpUtils.getUserName(httpServletRequest);
LOGGER.info("Start to get all tables with source database and target database.");
List<String> sourceTableName = new ArrayList<>();
List<String> targetTableName = new ArrayList<>();
if (request.getSourceLinkisDataSourceId() == null) {
GetUserTableByDbIdRequest sourceRequest = new GetUserTableByDbIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getSourceDb());
sourceRequest.setProxyUser(request.getProxyUser());
GeneralResponse<GetAllResponse<TableInfoDetail>> sourceTableInfos = getUserTableByDbId(sourceRequest);
if (sourceTableInfos.getData().getTotal() <= 0) {
throw new UnExpectedRequestException("Source database has no tables.");
}
sourceTableName.addAll(sourceTableInfos.getData().getData().stream().map(TableInfoDetail::getTableName).collect(Collectors.toList()));
} else {
Map response = getTablesByDataSource(request.getClusterName(), request.getProxyUser(), request.getSourceLinkisDataSourceId(), request.getSourceDb());
List<String> tables = (List<String>) response.get("tables");
for (String table : tables) {
sourceTableName.add(table);
}
}
if (request.getTargetLinkisDataSourceId() == null) {
GetUserTableByDbIdRequest targetRequest = new GetUserTableByDbIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getTargetDb());
targetRequest.setProxyUser(request.getProxyUser());
GeneralResponse<GetAllResponse<TableInfoDetail>> targetTableInfos = getUserTableByDbId(targetRequest);
if (targetTableInfos.getData().getTotal() <= 0) {
throw new UnExpectedRequestException("Target database has no tables.");
}
targetTableName.addAll(targetTableInfos.getData().getData().stream().map(TableInfoDetail::getTableName).collect(Collectors.toList()));
} else {
Map response = getTablesByDataSource(request.getClusterName(), request.getProxyUser(), request.getTargetLinkisDataSourceId(), request.getTargetDb());
List<String> tables = (List<String>) response.get("tables");
for (String table : tables) {
targetTableName.add(table);
}
}
// Collect the table with the same name that exists in the source database and the target database.
sourceTableName.retainAll(targetTableName);
Set<String> ruleFailedTable = new HashSet<>();
Long projectId = request.getProjectId();
List<String> ruleTables = new ArrayList<>(ruleDataSourceDao.findByProjectId(projectId).stream().map(RuleDataSource::getTableName).distinct().collect(Collectors.toList()));
// Filter tables for which rules have been created.
sourceTableName = sourceTableName.stream().filter(tableName -> !ruleTables.contains(tableName)).collect(Collectors.toList());
// Rule name index starts with the last created rule.
int ruleIndex = ruleDao.findByProject(projectDao.findById(projectId)).size();
// Filter tables with black list.
sourceTableName = filterTablesWithBlackList(request.getBlackList(), sourceTableName);
// White list.
List<String> whiteList = request.getWhiteList();
// Filter special table to compare with accuracy template.
List<FilterRequest> filterRequests = request.getFilterRequests();
List<FilterRequest> sameTableFilterRequests = filterRequests.stream().filter(filterRequest -> filterRequest.getSourceTable().equals(filterRequest.getTargetTable())).collect(Collectors.toList());
List<String> filterSameTableName = sameTableFilterRequests.stream().map(FilterRequest::getSourceTable).collect(Collectors.toList());
sourceTableName.removeAll(filterSameTableName);
filterRequests.removeAll(sameTableFilterRequests);
List<String> filterDiffTableName = filterRequests.stream().map(filterRequest -> filterRequest.getSourceTable() + SpecCharEnum.COLON.getValue() + filterRequest.getTargetTable()).collect(Collectors.toList());
whiteList.removeAll(filterDiffTableName);
filterRequests.addAll(sameTableFilterRequests);
// Check flag: create rules in a loop, and only do permission verification for the first time.
boolean check = true;
int sourceTableSize = sourceTableName.size();
int filterRequestsSize = filterRequests.size();
int total = ruleIndex + sourceTableSize + whiteList.size() + filterRequestsSize;
int i = ruleIndex;
for (; i < ruleIndex + sourceTableSize; i++) {
String currentSameTable = sourceTableName.get(i - ruleIndex);
try {
AddMultiSourceRuleRequest addMultiSourceRuleRequest = constructRequest(request, null, currentSameTable, currentSameTable, i, total, loginUser);
LOGGER.info("Start to add {}th multi source rule.", i);
if (i > 0) {
check = false;
}
multiSourceRuleService.addMultiSourceRule(addMultiSourceRuleRequest, check);
} catch (Exception e) {
LOGGER.error("One of rule failed to add, rule index:[{}], table name: [{}], exception:{}", i, currentSameTable, e.getMessage());
ruleFailedTable.add(currentSameTable);
}
LOGGER.info("Finish to add {}th multi source rule.", i);
}
// With white list.
withWhiteListAndFilterRequest(i, ruleIndex, sourceTableSize, total, whiteList, filterRequests, request, loginUser, check);
// Record the table that failed to create a rule.
saveRuleFailedTableInLabel(ruleFailedTable, projectId);
// projectEventService.record(projectId, loginUser, "create multi-source rule with dbs", "Success: " + (total - ruleFailedTable.size()) + "; Failed: " + ruleFailedTable.size(), EventTypeEnum.MODIFY_PROJECT.getCode());
return request.getSourceDb() + " vs " + request.getSourceDb() + ": " + "{&CONTINUE_CREATE_RULES}";
}
Aggregations