use of com.webank.wedatasphere.qualitis.metadata.response.table.TableStatisticsInfo in project Qualitis by WeBankFinTech.
the class ExecutionManagerImpl method executeFileRule.
@Override
public TaskSubmitResult executeFileRule(List<Rule> fileRules, String submitTime, Application application, String user, String clusterName, StringBuffer runDate) throws UnExpectedRequestException, MetaDataAcquireFailedException {
LOGGER.info("Start to execute file rule task and save check result.");
Task taskInDb = taskDao.save(new Task(application, submitTime, TaskStatusEnum.SUBMITTED.getCode()));
Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
int totalRules = fileRules.size();
int successRule = 0;
for (Rule rule : fileRules) {
if (rule.getAbortOnFailure() != null) {
taskInDb.setAbortOnFailure(rule.getAbortOnFailure());
}
TaskRuleSimple taskRuleSimple = new TaskRuleSimple(rule, taskInDb, httpServletRequest.getHeader("Content-Language"));
taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
// Check rule datasource: 1) table 2) partition.
if (StringUtils.isEmpty(ruleDataSource.getFilter())) {
TableStatisticsInfo result;
try {
String proxyUser = ruleDataSource.getProxyUser();
result = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
} catch (RestClientException e) {
LOGGER.error("Failed to get table statistics with linkis api.", e);
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
if (result == null) {
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
String fullSize = result.getTableSize();
List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getTableFileCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
} else {
PartitionStatisticsInfo result;
try {
String proxyUser = ruleDataSource.getProxyUser();
result = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(DateExprReplaceUtil.replaceFilter(new Date(), ruleDataSource.getFilter())), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
} catch (RestClientException e) {
LOGGER.error("Failed to get table statistics with linkis api.", e);
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
if (result == null) {
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
String fullSize = result.getPartitionSize();
List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getPartitionChildCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
}
if (taskInDb.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
break;
}
}
taskInDb.setTaskDataSources(taskDataSources);
taskInDb.setTaskRuleSimples(taskRuleSimples);
// Update task status
taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
if (totalRules == successRule) {
taskInDb.setStatus(TaskStatusEnum.PASS_CHECKOUT.getCode());
taskInDb.setProgress(Double.parseDouble("1"));
}
taskDao.save(taskInDb);
LOGGER.info("Finished to execute file rule task and save check result.");
TaskSubmitResult taskSubmitResult = new TaskSubmitResult();
taskSubmitResult.setApplicationId(application.getId());
taskSubmitResult.setClusterName(clusterName);
return taskSubmitResult;
}
use of com.webank.wedatasphere.qualitis.metadata.response.table.TableStatisticsInfo in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method checkDatasource.
private void checkDatasource(Rule currentRule, String userName, StringBuffer partition, List<Map<String, String>> mappingCols, String nodeName, String clusterName, Map<Long, Map> dataSourceMysqlConnect) throws UnExpectedRequestException, MetaDataAcquireFailedException, DataSourceOverSizeException, DataSourceMoveException, BothNullDatasourceException, LeftNullDatasourceException, RightNullDatasourceException {
// For multi source rule to check tables' size before submit.
List<Double> datasourceSizeList = new ArrayList<>(currentRule.getRuleDataSources().size());
for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
Map<String, String> mappingCol = null;
if (ORIGINAL_INDEX.equals(ruleDataSource.getDatasourceIndex())) {
continue;
}
if (ruleDataSource.getDatasourceIndex() != null && mappingCols.get(ruleDataSource.getDatasourceIndex()).size() > 0) {
mappingCol = mappingCols.get(ruleDataSource.getDatasourceIndex());
}
if (ruleDataSource.getLinkisDataSourceId() != null) {
LOGGER.info("Start to solve relationship datasource info.");
checkRdmsSqlMetaInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource, mappingCol);
GeneralResponse<Map> dataSourceInfoDetail = metaDataClient.getDataSourceInfoDetail(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource.getLinkisDataSourceId(), ruleDataSource.getLinkisDataSourceVersionId());
GeneralResponse<Map> dataSourceConnectParams = metaDataClient.getDataSourceConnectParams(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), userName, ruleDataSource.getLinkisDataSourceId(), ruleDataSource.getLinkisDataSourceVersionId());
Map connectParamsReal = (Map) dataSourceConnectParams.getData().get("connectParams");
if (connectParamsReal.size() == 0) {
throw new UnExpectedRequestException("{&THE_DATASOURCE_IS_NOT_DEPLOYED}");
}
Map connectParams = (Map) ((Map) dataSourceInfoDetail.getData().get("info")).get("connectParams");
String dataType = (String) ((Map) ((Map) dataSourceInfoDetail.getData().get("info")).get("dataSourceType")).get("name");
connectParams.put("dataType", dataType);
dataSourceMysqlConnect.put(ruleDataSource.getId(), connectParams);
continue;
}
// Parse filter fields.
List<String> filterFields = getFilterFields(partition.toString());
if (StringUtils.isNotBlank(ruleDataSource.getDbName()) && !ruleDataSource.getDbName().equals(RuleConstraintEnum.CUSTOM_DATABASE_PREFIS.getValue())) {
// Get actual fields info.
List<ColumnInfoDetail> cols = metaDataClient.getColumnInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), userName);
if (CollectionUtils.isEmpty(cols)) {
throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "]. {&RULE_DATASOURCE_BE_MOVED}");
}
// Get actual partition fields.
List<String> partitionFields = cols.stream().filter(ColumnInfoDetail::getPartitionField).map(ColumnInfoDetail::getFieldName).collect(Collectors.toList());
// Check filter fields.
boolean partitionTable = CollectionUtils.isNotEmpty(partitionFields);
if (partitionTable && partition.length() > 0) {
for (String filter : filterFields) {
if (!partitionFields.contains(filter)) {
throw new UnExpectedRequestException("Table[" + ruleDataSource.getTableName() + "]. {&THE_CHECK_FIELD_DOES_NOT_EXIST_IN_PARTITIONS}[" + filter + "]");
}
}
// Check partition size.
PartitionStatisticsInfo partitionStatisticsInfo = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(partition.toString()), userName);
String fullSize = partitionStatisticsInfo.getPartitionSize();
ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
if (clusterInfo != null && StringUtils.isNotBlank(clusterInfo.getSkipDataSize()) && StringUtils.isNotBlank(fullSize)) {
double number = 0;
String unit = "B";
if (!"0B".equals(fullSize)) {
number = Double.parseDouble(fullSize.split(" ")[0]);
unit = fullSize.split(" ")[1];
}
datasourceSizeList.add(number);
String[] skipDataSize = clusterInfo.getSkipDataSize().split(" ");
double res = UnitTransfer.alarmconfigToTaskResult(number, skipDataSize[1], unit);
LOGGER.info("Check datasource[" + fullSize + "] if or not oversize with system config[" + clusterInfo.getSkipDataSize() + "]");
if (res > Double.parseDouble(skipDataSize[0])) {
throw new DataSourceOverSizeException("Table[" + ruleDataSource.getTableName() + "]. {&TABLE_IS_OVERSIZE_WITH_SYSTEM_CONFIG}:[" + clusterInfo.getSkipDataSize() + "]");
}
}
} else {
// Check table size.
TableStatisticsInfo tableStatisticsInfo = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), userName);
String fullSize = tableStatisticsInfo.getTableSize();
if (NULL_TABLE_SIZE.equals(fullSize)) {
throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "] {&RULE_DATASOURCE_BE_MOVED}");
}
ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName());
if (clusterInfo != null && StringUtils.isNotBlank(clusterInfo.getSkipDataSize()) && StringUtils.isNotBlank(fullSize)) {
LOGGER.info("Check datasource[" + fullSize + "] if or not oversize with system config[" + clusterInfo.getSkipDataSize() + "]");
double number = 0;
String unit = "B";
if (!"0B".equals(fullSize)) {
number = Double.parseDouble(fullSize.split(" ")[0]);
unit = fullSize.split(" ")[1];
}
datasourceSizeList.add(number);
String[] skipDataSize = clusterInfo.getSkipDataSize().split(" ");
double res = UnitTransfer.alarmconfigToTaskResult(number, skipDataSize[1], unit);
if (res > Double.parseDouble(skipDataSize[0])) {
throw new DataSourceOverSizeException("Table[" + ruleDataSource.getTableName() + "] is oversize with system config:[" + clusterInfo.getSkipDataSize() + "]");
}
}
partition.delete(0, partition.length());
}
if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
continue;
}
if (!metaDataClient.fieldExist(ruleDataSource.getColName(), cols, mappingCol)) {
throw new DataSourceMoveException("Table[" + ruleDataSource.getTableName() + "] {&RULE_DATASOURCE_BE_MOVED}");
}
} else if (StringUtils.isNotBlank(currentRule.getCsId())) {
checkDatasourceInContextService(ruleDataSource, mappingCol, clusterName, userName, nodeName, currentRule.getCsId());
}
}
if (CollectionUtils.isNotEmpty(datasourceSizeList) && currentRule.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())) {
double left = datasourceSizeList.get(0);
double right = datasourceSizeList.get(1);
LOGGER.info("Current multi source rule left table size number[{}], right table size number[{}]", left, right);
if (left == 0 && right == 0) {
throw new BothNullDatasourceException("{&BOTH_SIDE_ARE_NULL}");
} else if (left == 0) {
throw new LeftNullDatasourceException("{&ONE_SIDE_ARE_NULL}");
} else if (right == 0) {
throw new RightNullDatasourceException("{&ONE_SIDE_ARE_NULL}");
}
}
}
use of com.webank.wedatasphere.qualitis.metadata.response.table.TableStatisticsInfo in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method submitRulesWithDynamicPartition.
private void submitRulesWithDynamicPartition(List<ApplicationSubmitRequest> applicationSubmitRequests, Long projectId, Long ruleGroupId, List<Rule> rules, List<Long> ruleIds, String executionUser, boolean dynamicPartition, String clusterName, StringBuffer partition, String dynamicPartitionPrefix) throws UnExpectedRequestException, NoPartitionException {
if (dynamicPartition) {
for (Rule rule : rules) {
RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
if (ruleDataSource == null) {
throw new UnExpectedRequestException("Rule datasource has been broken.");
}
if (RuleTypeEnum.CUSTOM_RULE.getCode().equals(rule.getRuleType())) {
throw new UnExpectedRequestException(RuleTypeEnum.CUSTOM_RULE.getMessage() + " {&IS_NOT_SUPPORT}");
}
List<Map> currentPartitionMap;
try {
TableStatisticsInfo tableStatisticsInfo = metaDataClient.getTableStatisticsInfo(StringUtils.isBlank(clusterName) ? ruleDataSource.getClusterName() : clusterName, ruleDataSource.getDbName(), ruleDataSource.getTableName(), executionUser);
currentPartitionMap = tableStatisticsInfo.getPartitions();
if (StringUtils.isNotEmpty(dynamicPartitionPrefix)) {
metaDataClient.getPartitionStatisticsInfo(StringUtils.isBlank(clusterName) ? ruleDataSource.getClusterName() : clusterName, ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(dynamicPartitionPrefix), executionUser);
}
} catch (MetaDataAcquireFailedException e) {
LOGGER.error("Dynamic submit failed.", e);
throw new NoPartitionException(ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + "{&HAS_NO_PARTITIONS_TO_BE_EXECUTE}");
}
List<String> partitionList = new ArrayList<>();
if (CollectionUtils.isNotEmpty(currentPartitionMap)) {
ruleIds.remove(rule.getId());
getPartitionListAndSubmit(currentPartitionMap, partitionList, rule, partition, filterToPartitionPath(dynamicPartitionPrefix), applicationSubmitRequests, projectId, ruleGroupId);
} else {
throw new NoPartitionException(ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + "{&HAS_NO_PARTITIONS_TO_BE_EXECUTE}");
}
}
}
}
use of com.webank.wedatasphere.qualitis.metadata.response.table.TableStatisticsInfo in project Qualitis by WeBankFinTech.
the class MetaDataClientImpl method getTableStatisticsInfo.
@Override
public TableStatisticsInfo getTableStatisticsInfo(String clusterName, String dbName, String tableName, String userName) throws UnExpectedRequestException, MetaDataAcquireFailedException, RestClientException {
ClusterInfo clusterInfo = checkClusterNameExists(clusterName);
// send request to get dbs
String url = getPath(clusterInfo.getLinkisAddress()).path(linkisConfig.getTableStatistics()).queryParam("database", dbName).queryParam("tableName", tableName).toString();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.add("Token-User", userName);
headers.add("Token-Code", clusterInfo.getLinkisToken());
HttpEntity<Object> entity = new HttpEntity<>(headers);
LOGGER.info("Start to get table info by linkis. url: {}, method: {}, body: {}", url, javax.ws.rs.HttpMethod.GET, entity);
Map<String, Object> response;
try {
response = restTemplate.exchange(url, HttpMethod.GET, entity, Map.class).getBody();
} catch (ResourceAccessException e) {
LOGGER.error(e.getMessage(), e);
throw new MetaDataAcquireFailedException("Error! Can not get table info from linkis, exception: " + e.getMessage(), 500);
}
LOGGER.info("Finish to get table info by linkis. response: {}", response);
if (!checkResponse(response)) {
String message = (String) response.get("message");
LOGGER.error("Error! Can not get meta data from linkis, message: " + message);
throw new MetaDataAcquireFailedException("Error! Can not get table info from linkis, exception: " + message);
}
Map<String, Object> result = (Map<String, Object>) ((Map<String, Object>) response.get("data")).get("tableStatisticInfo");
TableStatisticsInfo tableStatisticsInfo = new TableStatisticsInfo();
tableStatisticsInfo.setTableFileCount(Integer.parseInt(result.get("fileNum").toString()));
tableStatisticsInfo.setTableSize(result.get("tableSize").toString());
tableStatisticsInfo.setPartitions((List<Map>) result.get("partitions"));
return tableStatisticsInfo;
}
Aggregations