use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class IndexServiceImpl method getPerDayApplications.
private Map<String, List<Application>> getPerDayApplications(List<Application> applications) {
Map<String, List<Application>> perDayAppsMap = new HashMap<>(applications.size());
for (Application application : applications) {
// Get date of time
String date = application.getSubmitTime().substring(0, 10);
if (perDayAppsMap.containsKey(date)) {
perDayAppsMap.get(date).add(application);
continue;
}
List<Application> applicationList = new ArrayList<>();
applicationList.add(application);
perDayAppsMap.put(date, applicationList);
}
return perDayAppsMap;
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class ApplicationServiceImpl method filterApplicationId.
/**
* Find application by applicationId
* @param applicationId
* @return
*/
@Override
public GeneralResponse<?> filterApplicationId(String applicationId) {
Long userId = HttpUtils.getUserId(httpServletRequest);
// Find applications by user
User user = userDao.findById(userId);
List<Application> applicationList;
applicationList = applicationDao.findByCreateUserAndId(user.getUserName(), applicationId);
if (applicationList == null) {
LOGGER.info("User: {} , Not find applications with applicationId: {}", user.getUserName(), applicationId);
return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS_BUT_FIND_NO_RESULTS}", null);
}
GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
List<ApplicationResponse> applicationResponses = new ArrayList<>();
for (Application application : applicationList) {
List<Task> tasks = taskDao.findByApplication(application);
ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
if (application.getCreateUser().equals(user.getUserName()) || application.getExecuteUser().equals(user.getUserName())) {
response.setKillOption(true);
} else {
response.setKillOption(false);
}
applicationResponses.add(response);
}
getAllResponse.setData(applicationResponses);
getAllResponse.setTotal(applicationList.size());
List<String> applicationIdList = getAllResponse.getData().stream().map(ApplicationResponse::getApplicationId).collect(Collectors.toList());
LOGGER.info("User: {}, find {} applications with like applicationId : {},Id of applications: {}", user.getUserName(), applicationList.size(), applicationId, applicationIdList);
return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class ApplicationServiceImpl method filterAdvanceApplication.
@Override
public GeneralResponse<?> filterAdvanceApplication(FilterAdvanceRequest request) {
Long userId = HttpUtils.getUserId(httpServletRequest);
// Find applications by user
User user = userDao.findById(userId);
List<Application> applicationList;
long total = 0;
// If application ID is not empty, just return application which ID like the input string.
if (StringUtils.isNotBlank(request.getApplicationId())) {
applicationList = applicationDao.findByCreateUserAndId(user.getUserName(), request.getApplicationId());
total = applicationList.size();
} else if (StringUtils.isNotBlank(request.getClusterName())) {
if (request.getStatus() != null) {
if (request.getStatus().equals(ApplicationStatusEnum.FINISHED.getCode())) {
request.setStatus(TaskStatusEnum.PASS_CHECKOUT.getCode());
} else if (request.getStatus().equals(ApplicationStatusEnum.NOT_PASS.getCode())) {
request.setStatus(TaskStatusEnum.FAIL_CHECKOUT.getCode());
} else if (request.getStatus().equals(ApplicationStatusEnum.SUCCESSFUL_CREATE_APPLICATION.getCode())) {
request.setStatus(TaskStatusEnum.INITED.getCode());
} else if (request.getStatus().equals(ApplicationStatusEnum.RUNNING.getCode())) {
request.setStatus(TaskStatusEnum.RUNNING.getCode());
} else if (request.getStatus() == 0) {
request.setStatus(null);
}
}
// If data source is not empty, it will be used as the basic filter.
applicationList = applicationDao.findApplicationByAdavnceConditionsWithDatasource(user.getUserName(), request.getClusterName(), request.getDatabaseName(), request.getTableName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime(), request.getPage(), request.getSize());
total = applicationDao.countApplicationByAdavnceConditionsWithDatasource(user.getUserName(), request.getClusterName(), request.getDatabaseName(), request.getTableName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime());
} else {
applicationList = applicationDao.findApplicationByAdavnceConditions(user.getUserName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime(), request.getPage(), request.getSize());
total = applicationDao.countApplicationByAdavnceConditions(user.getUserName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime());
}
GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
List<ApplicationResponse> applicationResponses = new ArrayList<>();
for (Application application : applicationList) {
List<Task> tasks = taskDao.findByApplication(application);
ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
if (application.getCreateUser().equals(user.getUserName()) || application.getExecuteUser().equals(user.getUserName())) {
response.setKillOption(true);
} else {
response.setKillOption(false);
}
applicationResponses.add(response);
}
getAllResponse.setData(applicationResponses);
getAllResponse.setTotal(total);
return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class ApplicationServiceImpl method filterDataSourceApplication.
@Override
public GeneralResponse<?> filterDataSourceApplication(FilterDataSourceRequest request) throws UnExpectedRequestException {
// Check arguments
FilterDataSourceRequest.checkRequest(request);
Integer page = request.getPage();
Integer size = request.getSize();
String clusterName = request.getClusterName();
String databaseName = StringUtils.isEmpty(request.getDatabaseName()) ? "" : request.getDatabaseName();
String tableName = StringUtils.isEmpty(request.getTableName()) ? "" : request.getTableName();
Long userId = HttpUtils.getUserId(httpServletRequest);
User user = userDao.findById(userId);
List<TaskDataSource> taskDataSources;
long total;
// Find datasource by user
taskDataSources = taskDataSourceDao.findByCreateUserAndDatasource(user.getUserName(), clusterName, databaseName, tableName, page, size);
total = taskDataSourceDao.countByCreateUserAndDatasource(user.getUserName(), clusterName, databaseName, tableName);
List<Application> applicationList = taskDataSources.stream().map(jobDataSource -> jobDataSource.getTask().getApplication()).collect(Collectors.toList());
GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
List<ApplicationResponse> applicationResponses = new ArrayList<>();
for (Application application : applicationList) {
List<Task> tasks = taskDao.findByApplication(application);
ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
if (application.getCreateUser().equals(user.getUserName()) || application.getExecuteUser().equals(user.getUserName())) {
response.setKillOption(true);
} else {
response.setKillOption(false);
}
applicationResponses.add(response);
}
getAllResponse.setData(applicationResponses);
getAllResponse.setTotal(total);
List<String> applicationIdList = getAllResponse.getData().stream().map(ApplicationResponse::getApplicationId).collect(Collectors.toList());
LOGGER.info("Succeed to find applications. size: {}, id of applications: {}", total, applicationIdList);
return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}
use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.
the class OuterExecutionServiceImpl method commonExecution.
@Override
public ApplicationTaskSimpleResponse commonExecution(List<Rule> rules, StringBuffer partition, String executionUser, String nodeName, String startupParam, String clusterName, String setFlag, Map<String, String> execParams, Application newApplication, Date date, StringBuffer runDate) throws RuleVariableNotSupportException, JobSubmitException, RuleVariableNotFoundException, ArgumentException, ConvertException, DataQualityTaskException, TaskTypeException, ClusterInfoNotConfigException, SystemConfigException, UnExpectedRequestException, MetaDataAcquireFailedException, IOException, TaskNotExistException, SemanticException, ParseException, DataSourceMoveException, DataSourceOverSizeException, org.apache.hadoop.hive.ql.parse.ParseException, BothNullDatasourceException, RightNullDatasourceException, LeftNullDatasourceException, java.text.ParseException {
// current user
String userName = executionUser;
// Generate database name.
String database = generateDatabase(userName);
LOGGER.info("Succeed to generate database_name: {}", database);
// Save application
newApplication.setRuleSize(rules.size());
newApplication.setSavedDb(database);
Application saveApplication = applicationDao.saveApplication(newApplication);
List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
List<Rule> fileRules = new ArrayList<>();
Map<Long, Map> dataSourceMysqlConnect = new HashMap<>(2);
for (Iterator<Rule> iterator = rules.iterator(); iterator.hasNext(); ) {
Rule currentRule = iterator.next();
if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
// Replace with execution parameter and parse datasource to save.
customReSaveDateSource(currentRule, execParams, clusterName, date);
}
List<Map<String, String>> mappingCols = new ArrayList<>();
getMappingCols(currentRule, mappingCols);
// Check datasource before submit job.
try {
checkDatasource(currentRule, userName, partition, mappingCols, nodeName, clusterName, dataSourceMysqlConnect);
} catch (BothNullDatasourceException e) {
Task taskInDb = taskDao.save(new Task(newApplication, newApplication.getSubmitTime(), TaskStatusEnum.SUCCEED.getCode()));
taskInDb.setClusterName(clusterName);
TaskRuleSimple taskRuleSimple = new TaskRuleSimple(currentRule, taskInDb);
Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
}
taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
taskInDb.setTaskDataSources(taskDataSources);
taskInDb.setTaskRuleSimples(taskRuleSimples);
TaskResult taskResult = new TaskResult();
taskResult.setApplicationId(newApplication.getId());
taskResult.setCreateTime(newApplication.getSubmitTime());
taskResult.setRuleId(currentRule.getId());
taskResult.setResultType("Long");
taskResult.setValue(0 + "");
taskResultDao.saveTaskResult(taskResult);
taskDao.save(taskInDb);
iterator.remove();
if (!iterator.hasNext()) {
throw e;
}
continue;
}
if (currentRule.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())) {
fileRules.add(currentRule);
LOGGER.info("Succeed to find file rule. Rule: {}", currentRule.getId() + " " + currentRule.getName());
iterator.remove();
}
}
String submitTime = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(date);
// General task.
if (!rules.isEmpty()) {
taskSubmitResults.addAll(executionManager.submitApplication(rules, nodeName, submitTime, userName, database, partition, date, saveApplication, clusterName, startupParam, setFlag, execParams, runDate, dataSourceMysqlConnect));
}
// Execute file rule task and save task result.
if (!fileRules.isEmpty()) {
taskSubmitResults.add(executionManager.executeFileRule(fileRules, submitTime, saveApplication, userName, clusterName, runDate));
}
saveApplication.setTotalTaskNum(taskSubmitResults.size());
LOGGER.info("Succeed to submit application. result: {}", taskSubmitResults);
Application applicationInDb = applicationDao.saveApplication(saveApplication);
LOGGER.info("Succeed to save application. application: {}", applicationInDb);
return new ApplicationTaskSimpleResponse(taskSubmitResults);
}
Aggregations