use of com.webank.wedatasphere.qualitis.entity.TaskDataSource in project Qualitis by WeBankFinTech.
the class ExecutionManagerImpl method saveJobRuleSimpleAndJobDataSource.
/**
* Save task rule simple and rule datasource
* @param ruleList
* @param task
*/
private void saveJobRuleSimpleAndJobDataSource(List<TaskRule> ruleList, Task task) {
for (TaskRule rule : ruleList) {
if (rule.getChildRuleId() != null) {
// Save parent task rule simple
TaskRuleSimple parentRuleSimple = taskRuleSimpleRepository.save(new TaskRuleSimple(rule, task, true, null));
// Save child task rule simple
TaskRuleSimple childRuleSimple = taskRuleSimpleRepository.save(new TaskRuleSimple(rule, task, false, parentRuleSimple));
task.getTaskRuleSimples().add(parentRuleSimple);
} else {
TaskRuleSimple taskRuleSimple = taskRuleSimpleRepository.save(new TaskRuleSimple(rule, task));
LOGGER.info("Succeed to save task_rule: task_rule_id: {}, rule_name: {}", taskRuleSimple.getId(), taskRuleSimple.getRuleName());
task.getTaskRuleSimples().add(taskRuleSimple);
}
for (TaskRuleDataSource ruleDataSource : rule.getTaskRuleDataSourceList()) {
TaskDataSource taskDataSource = taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, task));
LOGGER.info("Succeed to save task_datasource: task_datasource_id: {}, cluster: {}, database: {}, table: {}", taskDataSource.getId(), taskDataSource.getClusterName(), taskDataSource.getDatabaseName(), taskDataSource.getTableName());
task.getTaskDataSources().add(taskDataSource);
}
}
}
use of com.webank.wedatasphere.qualitis.entity.TaskDataSource in project Qualitis by WeBankFinTech.
the class ExecutionManagerImpl method executeFileRule.
@Override
public TaskSubmitResult executeFileRule(List<Rule> fileRules, String submitTime, Application application, String user, String clusterName, StringBuffer runDate) throws UnExpectedRequestException, MetaDataAcquireFailedException {
LOGGER.info("Start to execute file rule task and save check result.");
Task taskInDb = taskDao.save(new Task(application, submitTime, TaskStatusEnum.SUBMITTED.getCode()));
Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
int totalRules = fileRules.size();
int successRule = 0;
for (Rule rule : fileRules) {
if (rule.getAbortOnFailure() != null) {
taskInDb.setAbortOnFailure(rule.getAbortOnFailure());
}
TaskRuleSimple taskRuleSimple = new TaskRuleSimple(rule, taskInDb, httpServletRequest.getHeader("Content-Language"));
taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
RuleDataSource ruleDataSource = rule.getRuleDataSources().iterator().next();
taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
// Check rule datasource: 1) table 2) partition.
if (StringUtils.isEmpty(ruleDataSource.getFilter())) {
TableStatisticsInfo result;
try {
String proxyUser = ruleDataSource.getProxyUser();
result = metaDataClient.getTableStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
} catch (RestClientException e) {
LOGGER.error("Failed to get table statistics with linkis api.", e);
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
if (result == null) {
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
String fullSize = result.getTableSize();
List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getTableFileCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
} else {
PartitionStatisticsInfo result;
try {
String proxyUser = ruleDataSource.getProxyUser();
result = metaDataClient.getPartitionStatisticsInfo(StringUtils.isNotBlank(clusterName) ? clusterName : ruleDataSource.getClusterName(), ruleDataSource.getDbName(), ruleDataSource.getTableName(), filterToPartitionPath(DateExprReplaceUtil.replaceFilter(new Date(), ruleDataSource.getFilter())), StringUtils.isNotBlank(proxyUser) ? proxyUser : user);
} catch (RestClientException e) {
LOGGER.error("Failed to get table statistics with linkis api.", e);
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
if (result == null) {
throw new UnExpectedRequestException("{&FAILED_TO_GET_DATASOURCE_INFO}");
}
String fullSize = result.getPartitionSize();
List<TaskResult> taskResultInDbs = saveTaskRusult(fullSize, Double.parseDouble(result.getPartitionChildCount() + ""), application, submitTime, rule, rule.getAlarmConfigs(), runDate.toString());
successRule = modifyTaskStatus(taskRuleSimple.getTaskRuleAlarmConfigList(), taskInDb, taskResultInDbs, successRule);
}
if (taskInDb.getStatus().equals(TaskStatusEnum.FAILED.getCode())) {
break;
}
}
taskInDb.setTaskDataSources(taskDataSources);
taskInDb.setTaskRuleSimples(taskRuleSimples);
// Update task status
taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
if (totalRules == successRule) {
taskInDb.setStatus(TaskStatusEnum.PASS_CHECKOUT.getCode());
taskInDb.setProgress(Double.parseDouble("1"));
}
taskDao.save(taskInDb);
LOGGER.info("Finished to execute file rule task and save check result.");
TaskSubmitResult taskSubmitResult = new TaskSubmitResult();
taskSubmitResult.setApplicationId(application.getId());
taskSubmitResult.setClusterName(clusterName);
return taskSubmitResult;
}
use of com.webank.wedatasphere.qualitis.entity.TaskDataSource in project Qualitis by WeBankFinTech.
the class PredicateTest method test.
@Test
@Transactional
public void test() {
String user = "v_wblwyan_test1";
/*
* ProjectUserDao测试
*/
DataSourceQo param = new DataSourceQo();
param.setUser(user);
param.setUserType(new Integer[] { ProjectUserPermissionEnum.CREATOR.getCode() });
List<ProjectUser> projectUsers = dao.findByUsernameAndPermissionsIn(param);
assertTrue(projectUsers.isEmpty());
param.setUserType(new Integer[] { ProjectUserPermissionEnum.CREATOR.getCode() });
projectUsers = dao.findByUsernameAndPermissionsIn(param);
assertTrue(projectUsers.isEmpty());
param.setUserType(null);
projectUsers = dao.findByUsernameAndPermissionsIn(param);
assertTrue(projectUsers.isEmpty());
param.setUserType(new Integer[] {});
projectUsers = dao.findByUsernameAndPermissionsIn(param);
assertTrue(projectUsers.isEmpty());
/*
*TaskDataSourceDao 测试
*/
List<TaskDataSource> taskDataSources = taskDataSourceDao.findByUserAndDataSource(user, null, null, null, 0, 5);
assertTrue(taskDataSources.isEmpty());
taskDataSources = taskDataSourceDao.findByUserAndDataSource(user, "clusterName_test", null, null, 0, 5);
assertTrue(taskDataSources.isEmpty());
taskDataSources = taskDataSourceDao.findByUserAndDataSource(user, "clusterName_test", "databaseName_test", null, 0, 5);
assertTrue(taskDataSources.isEmpty());
taskDataSources = taskDataSourceDao.findByUserAndDataSource(user, "clusterName_test", "databaseName_test", "tableName_test", 0, 5);
assertTrue(taskDataSources.isEmpty());
long count = taskDataSourceDao.countByUserAndDataSource(user, "clusterName_test", "databaseName_test", null);
assertEquals(0, count);
/*
*RuleDataSourceDao 测试
*/
List<RuleDataSource> ruleDataSources = ruleDataSourceDao.findByProjectUser((long) Integer.MAX_VALUE, null, null, null);
assertTrue(ruleDataSources.isEmpty());
ruleDataSources = ruleDataSourceDao.findByProjectUser((long) Integer.MAX_VALUE, "clusterName_test", null, null);
assertTrue(ruleDataSources.isEmpty());
ruleDataSources = ruleDataSourceDao.findByProjectUser((long) Integer.MAX_VALUE, "clusterName_test", "databaseName_test", null);
assertTrue(ruleDataSources.isEmpty());
ruleDataSources = ruleDataSourceDao.findByProjectUser((long) Integer.MAX_VALUE, "clusterName_test", "databaseName_test", "tableName_test");
assertTrue(ruleDataSources.isEmpty());
}
use of com.webank.wedatasphere.qualitis.entity.TaskDataSource in project Qualitis by WeBankFinTech.
the class ApplicationServiceImpl method filterDataSourceApplication.
@Override
public GeneralResponse<?> filterDataSourceApplication(FilterDataSourceRequest request) throws UnExpectedRequestException {
// Check arguments
FilterDataSourceRequest.checkRequest(request);
Integer page = request.getPage();
Integer size = request.getSize();
String clusterName = request.getClusterName();
String databaseName = StringUtils.isEmpty(request.getDatabaseName()) ? "" : request.getDatabaseName();
String tableName = StringUtils.isEmpty(request.getTableName()) ? "" : request.getTableName();
Long userId = HttpUtils.getUserId(httpServletRequest);
User user = userDao.findById(userId);
List<TaskDataSource> taskDataSources;
long total;
// Find datasource by user
taskDataSources = taskDataSourceDao.findByCreateUserAndDatasource(user.getUserName(), clusterName, databaseName, tableName, page, size);
total = taskDataSourceDao.countByCreateUserAndDatasource(user.getUserName(), clusterName, databaseName, tableName);
List<Application> applicationList = taskDataSources.stream().map(jobDataSource -> jobDataSource.getTask().getApplication()).collect(Collectors.toList());
GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
List<ApplicationResponse> applicationResponses = new ArrayList<>();
for (Application application : applicationList) {
List<Task> tasks = taskDao.findByApplication(application);
ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
if (application.getCreateUser().equals(user.getUserName()) || application.getExecuteUser().equals(user.getUserName())) {
response.setKillOption(true);
} else {
response.setKillOption(false);
}
applicationResponses.add(response);
}
getAllResponse.setData(applicationResponses);
getAllResponse.setTotal(total);
List<String> applicationIdList = getAllResponse.getData().stream().map(ApplicationResponse::getApplicationId).collect(Collectors.toList());
LOGGER.info("Succeed to find applications. size: {}, id of applications: {}", total, applicationIdList);
return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}
use of com.webank.wedatasphere.qualitis.entity.TaskDataSource in project Qualitis by WeBankFinTech.
the class TaskServiceImpl method getTaskDetail.
@Override
public GeneralResponse<?> getTaskDetail(Long taskId) throws UnExpectedRequestException {
Task taskInDb = taskDao.findById(taskId);
if (taskInDb == null) {
throw new UnExpectedRequestException("Job id [" + taskId + "] {&DOES_NOT_EXIST}");
}
// FIXME:
List<TaskRuleAlarmConfig> distinct;
Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(taskInDb.getTaskRuleSimples().size());
for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
distinct = taskRuleSimple.getTaskRuleAlarmConfigList().stream().distinct().collect(Collectors.toList());
taskRuleSimple.setTaskRuleAlarmConfigList(distinct);
taskRuleSimples.add(taskRuleSimple);
}
taskInDb.setTaskRuleSimples(taskRuleSimples);
// Find single table verification rules
List<TaskRuleSimple> singleRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.SINGLE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
// Find all datasources of single table verification rules, and save it in the map
Map<TaskRuleSimple, List<TaskDataSource>> singleRuleDataSourceMap = new HashMap<>(singleRuleIds.size());
for (TaskRuleSimple taskRuleSimple : singleRuleIds) {
singleRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
}
// Find custom table verification rules
List<TaskRuleSimple> customRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())).collect(Collectors.toList());
// Find all datasources of custom table verification rules, and save it in the map
Map<TaskRuleSimple, List<TaskDataSource>> customRuleDataSourceMap = new HashMap<>(customRuleIds.size());
for (TaskRuleSimple taskRuleSimple : customRuleIds) {
customRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
}
// Find multi-table verification rules
List<TaskRuleSimple> multiRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.MULTI_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
// Find all datasources of multi-table verification rules, and save it in the map
Map<TaskRuleSimple, List<TaskDataSource>> multiRuleDataSourceMap = new HashMap<>(multiRuleIds.size());
for (TaskRuleSimple taskRuleSimple : multiRuleIds) {
multiRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
}
// Find file table verification rules
List<TaskRuleSimple> fileRuleIds = taskInDb.getTaskRuleSimples().stream().filter(taskRuleSimple -> taskRuleSimple.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())).collect(Collectors.toList());
// Find all datasources of file table verification rules, and save it in the map
Map<TaskRuleSimple, List<TaskDataSource>> fileRuleDataSourceMap = new HashMap<>(fileRuleIds.size());
for (TaskRuleSimple taskRuleSimple : fileRuleIds) {
fileRuleDataSourceMap.put(taskRuleSimple, taskDataSourceDao.findByTaskAndRuleId(taskInDb, taskRuleSimple.getRuleId()));
}
List<Long> allRuleIds = new ArrayList<>();
for (TaskRuleSimple taskRuleSimple : taskInDb.getTaskRuleSimples()) {
allRuleIds.add(taskRuleSimple.getRuleId());
if (taskRuleSimple.getChildRuleSimple() != null) {
allRuleIds.add(taskRuleSimple.getChildRuleSimple().getRuleId());
}
}
List<TaskResult> allTaskResult = taskResultDao.findByApplicationIdAndRuleIn(taskInDb.getApplication().getId(), allRuleIds);
Map<Long, List<TaskResult>> allResultMap = new HashMap<>(allTaskResult.size());
for (TaskResult taskResult : allTaskResult) {
if (allResultMap.get(taskResult.getRuleId()) != null) {
allResultMap.get(taskResult.getRuleId()).add(taskResult);
} else {
List<TaskResult> taskResults = new ArrayList<>(1);
taskResults.add(taskResult);
allResultMap.put(taskResult.getRuleId(), taskResults);
}
}
TaskCheckResultResponse taskCheckResultResponse = new TaskCheckResultResponse(taskInDb, singleRuleDataSourceMap, customRuleDataSourceMap, multiRuleDataSourceMap, fileRuleDataSourceMap, allResultMap);
return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_TASK_DETAIL}", taskCheckResultResponse);
}
Aggregations