Search in sources :

Example 11 with Application

use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.

the class IndexServiceImpl method getPerDayApplications.

private Map<String, List<Application>> getPerDayApplications(List<Application> applications) {
    Map<String, List<Application>> perDayAppsMap = new HashMap<>(applications.size());
    for (Application application : applications) {
        // Get date of time
        String date = application.getSubmitTime().substring(0, 10);
        if (perDayAppsMap.containsKey(date)) {
            perDayAppsMap.get(date).add(application);
            continue;
        }
        List<Application> applicationList = new ArrayList<>();
        applicationList.add(application);
        perDayAppsMap.put(date, applicationList);
    }
    return perDayAppsMap;
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ArrayList(java.util.ArrayList) List(java.util.List) Application(com.webank.wedatasphere.qualitis.entity.Application)

Example 12 with Application

use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method filterApplicationId.

/**
 * Find application by applicationId
 * @param applicationId
 * @return
 */
@Override
public GeneralResponse<?> filterApplicationId(String applicationId) {
    Long userId = HttpUtils.getUserId(httpServletRequest);
    // Find applications by user
    User user = userDao.findById(userId);
    List<Application> applicationList;
    applicationList = applicationDao.findByCreateUserAndId(user.getUserName(), applicationId);
    if (applicationList == null) {
        LOGGER.info("User: {} , Not find applications with applicationId: {}", user.getUserName(), applicationId);
        return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS_BUT_FIND_NO_RESULTS}", null);
    }
    GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
    List<ApplicationResponse> applicationResponses = new ArrayList<>();
    for (Application application : applicationList) {
        List<Task> tasks = taskDao.findByApplication(application);
        ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
        if (application.getCreateUser().equals(user.getUserName()) || application.getExecuteUser().equals(user.getUserName())) {
            response.setKillOption(true);
        } else {
            response.setKillOption(false);
        }
        applicationResponses.add(response);
    }
    getAllResponse.setData(applicationResponses);
    getAllResponse.setTotal(applicationList.size());
    List<String> applicationIdList = getAllResponse.getData().stream().map(ApplicationResponse::getApplicationId).collect(Collectors.toList());
    LOGGER.info("User: {}, find {} applications with like applicationId : {},Id of applications: {}", user.getUserName(), applicationList.size(), applicationId, applicationIdList);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}
Also used : Task(com.webank.wedatasphere.qualitis.entity.Task) User(com.webank.wedatasphere.qualitis.entity.User) ApplicationResponse(com.webank.wedatasphere.qualitis.response.ApplicationResponse) ArrayList(java.util.ArrayList) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) GetAllResponse(com.webank.wedatasphere.qualitis.response.GetAllResponse) Application(com.webank.wedatasphere.qualitis.entity.Application)

Example 13 with Application

use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method filterAdvanceApplication.

@Override
public GeneralResponse<?> filterAdvanceApplication(FilterAdvanceRequest request) {
    Long userId = HttpUtils.getUserId(httpServletRequest);
    // Find applications by user
    User user = userDao.findById(userId);
    List<Application> applicationList;
    long total = 0;
    // If application ID is not empty, just return application which ID like the input string.
    if (StringUtils.isNotBlank(request.getApplicationId())) {
        applicationList = applicationDao.findByCreateUserAndId(user.getUserName(), request.getApplicationId());
        total = applicationList.size();
    } else if (StringUtils.isNotBlank(request.getClusterName())) {
        if (request.getStatus() != null) {
            if (request.getStatus().equals(ApplicationStatusEnum.FINISHED.getCode())) {
                request.setStatus(TaskStatusEnum.PASS_CHECKOUT.getCode());
            } else if (request.getStatus().equals(ApplicationStatusEnum.NOT_PASS.getCode())) {
                request.setStatus(TaskStatusEnum.FAIL_CHECKOUT.getCode());
            } else if (request.getStatus().equals(ApplicationStatusEnum.SUCCESSFUL_CREATE_APPLICATION.getCode())) {
                request.setStatus(TaskStatusEnum.INITED.getCode());
            } else if (request.getStatus().equals(ApplicationStatusEnum.RUNNING.getCode())) {
                request.setStatus(TaskStatusEnum.RUNNING.getCode());
            } else if (request.getStatus() == 0) {
                request.setStatus(null);
            }
        }
        // If data source is not empty, it will be used as the basic filter.
        applicationList = applicationDao.findApplicationByAdavnceConditionsWithDatasource(user.getUserName(), request.getClusterName(), request.getDatabaseName(), request.getTableName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime(), request.getPage(), request.getSize());
        total = applicationDao.countApplicationByAdavnceConditionsWithDatasource(user.getUserName(), request.getClusterName(), request.getDatabaseName(), request.getTableName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime());
    } else {
        applicationList = applicationDao.findApplicationByAdavnceConditions(user.getUserName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime(), request.getPage(), request.getSize());
        total = applicationDao.countApplicationByAdavnceConditions(user.getUserName(), request.getProjectId(), request.getStatus(), request.getCommentType(), request.getStartTime(), request.getEndTime());
    }
    GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
    List<ApplicationResponse> applicationResponses = new ArrayList<>();
    for (Application application : applicationList) {
        List<Task> tasks = taskDao.findByApplication(application);
        ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
        if (application.getCreateUser().equals(user.getUserName()) || application.getExecuteUser().equals(user.getUserName())) {
            response.setKillOption(true);
        } else {
            response.setKillOption(false);
        }
        applicationResponses.add(response);
    }
    getAllResponse.setData(applicationResponses);
    getAllResponse.setTotal(total);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}
Also used : Task(com.webank.wedatasphere.qualitis.entity.Task) User(com.webank.wedatasphere.qualitis.entity.User) ApplicationResponse(com.webank.wedatasphere.qualitis.response.ApplicationResponse) ArrayList(java.util.ArrayList) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) GetAllResponse(com.webank.wedatasphere.qualitis.response.GetAllResponse) Application(com.webank.wedatasphere.qualitis.entity.Application)

Example 14 with Application

use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.

the class ApplicationServiceImpl method filterDataSourceApplication.

@Override
public GeneralResponse<?> filterDataSourceApplication(FilterDataSourceRequest request) throws UnExpectedRequestException {
    // Check arguments
    FilterDataSourceRequest.checkRequest(request);
    Integer page = request.getPage();
    Integer size = request.getSize();
    String clusterName = request.getClusterName();
    String databaseName = StringUtils.isEmpty(request.getDatabaseName()) ? "" : request.getDatabaseName();
    String tableName = StringUtils.isEmpty(request.getTableName()) ? "" : request.getTableName();
    Long userId = HttpUtils.getUserId(httpServletRequest);
    User user = userDao.findById(userId);
    List<TaskDataSource> taskDataSources;
    long total;
    // Find datasource by user
    taskDataSources = taskDataSourceDao.findByCreateUserAndDatasource(user.getUserName(), clusterName, databaseName, tableName, page, size);
    total = taskDataSourceDao.countByCreateUserAndDatasource(user.getUserName(), clusterName, databaseName, tableName);
    List<Application> applicationList = taskDataSources.stream().map(jobDataSource -> jobDataSource.getTask().getApplication()).collect(Collectors.toList());
    GetAllResponse<ApplicationResponse> getAllResponse = new GetAllResponse<>();
    List<ApplicationResponse> applicationResponses = new ArrayList<>();
    for (Application application : applicationList) {
        List<Task> tasks = taskDao.findByApplication(application);
        ApplicationResponse response = new ApplicationResponse(application, tasks, httpServletRequest.getHeader("Content-Language"));
        if (application.getCreateUser().equals(user.getUserName()) || application.getExecuteUser().equals(user.getUserName())) {
            response.setKillOption(true);
        } else {
            response.setKillOption(false);
        }
        applicationResponses.add(response);
    }
    getAllResponse.setData(applicationResponses);
    getAllResponse.setTotal(total);
    List<String> applicationIdList = getAllResponse.getData().stream().map(ApplicationResponse::getApplicationId).collect(Collectors.toList());
    LOGGER.info("Succeed to find applications. size: {}, id of applications: {}", total, applicationIdList);
    return new GeneralResponse<>("200", "{&SUCCEED_TO_GET_APPLICATIONS}", getAllResponse);
}
Also used : HttpPost(org.apache.http.client.methods.HttpPost) AlarmConfigStatusEnum(com.webank.wedatasphere.qualitis.constant.AlarmConfigStatusEnum) StringUtils(org.apache.commons.lang.StringUtils) LinkisConfig(com.webank.wedatasphere.qualitis.config.LinkisConfig) LoggerFactory(org.slf4j.LoggerFactory) HttpStatus(org.apache.http.HttpStatus) Autowired(org.springframework.beans.factory.annotation.Autowired) Application(com.webank.wedatasphere.qualitis.entity.Application) ProjectUserDao(com.webank.wedatasphere.qualitis.project.dao.ProjectUserDao) ApplicationDatabaseResponse(com.webank.wedatasphere.qualitis.response.ApplicationDatabaseResponse) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) ClusterInfoDao(com.webank.wedatasphere.qualitis.dao.ClusterInfoDao) FilterDataSourceRequest(com.webank.wedatasphere.qualitis.request.FilterDataSourceRequest) HttpUtils(com.webank.wedatasphere.qualitis.util.HttpUtils) Task(com.webank.wedatasphere.qualitis.entity.Task) TaskRuleAlarmConfig(com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig) Map(java.util.Map) ApplicationClusterResponse(com.webank.wedatasphere.qualitis.response.ApplicationClusterResponse) UriBuilder(javax.ws.rs.core.UriBuilder) GetAllResponse(com.webank.wedatasphere.qualitis.response.GetAllResponse) Path(java.nio.file.Path) RestTemplate(org.springframework.web.client.RestTemplate) Context(javax.ws.rs.core.Context) MultipartEntityBuilder(org.apache.http.entity.mime.MultipartEntityBuilder) PageRequest(com.webank.wedatasphere.qualitis.request.PageRequest) UploadResultRequest(com.webank.wedatasphere.qualitis.request.UploadResultRequest) FilterAdvanceRequest(com.webank.wedatasphere.qualitis.request.FilterAdvanceRequest) ContentType(org.apache.http.entity.ContentType) UUID(java.util.UUID) Collectors(java.util.stream.Collectors) FileNotFoundException(java.io.FileNotFoundException) TaskRuleSimpleDao(com.webank.wedatasphere.qualitis.dao.TaskRuleSimpleDao) ExcelTypeEnum(com.alibaba.excel.support.ExcelTypeEnum) List(java.util.List) Sheet(com.alibaba.excel.metadata.Sheet) ClusterInfo(com.webank.wedatasphere.qualitis.entity.ClusterInfo) MetaDataClient(com.webank.wedatasphere.qualitis.metadata.client.MetaDataClient) UnExpectedRequestException(com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException) HttpClients(org.apache.http.impl.client.HttpClients) BeanUtils(org.springframework.beans.BeanUtils) TaskStatusEnum(com.webank.wedatasphere.qualitis.constant.TaskStatusEnum) ApplicationResponse(com.webank.wedatasphere.qualitis.response.ApplicationResponse) HashMap(java.util.HashMap) FilterProjectRequest(com.webank.wedatasphere.qualitis.request.FilterProjectRequest) FilterStatusRequest(com.webank.wedatasphere.qualitis.request.FilterStatusRequest) ProjectDao(com.webank.wedatasphere.qualitis.project.dao.ProjectDao) ArrayList(java.util.ArrayList) ExcelWriter(com.alibaba.excel.ExcelWriter) HttpServletRequest(javax.servlet.http.HttpServletRequest) User(com.webank.wedatasphere.qualitis.entity.User) CheckTemplateEnum(com.webank.wedatasphere.qualitis.rule.constant.CheckTemplateEnum) Charset(java.nio.charset.Charset) TaskDao(com.webank.wedatasphere.qualitis.dao.TaskDao) CloseableHttpResponse(org.apache.http.client.methods.CloseableHttpResponse) Service(org.springframework.stereotype.Service) ApplicationStatusEnum(com.webank.wedatasphere.qualitis.constant.ApplicationStatusEnum) CompareTypeEnum(com.webank.wedatasphere.qualitis.rule.constant.CompareTypeEnum) ApplicationService(com.webank.wedatasphere.qualitis.service.ApplicationService) TaskResult(com.webank.wedatasphere.qualitis.entity.TaskResult) OutputStream(java.io.OutputStream) CloseableHttpClient(org.apache.http.impl.client.CloseableHttpClient) Logger(org.slf4j.Logger) Files(java.nio.file.Files) RuleMetric(com.webank.wedatasphere.qualitis.entity.RuleMetric) TaskDataSourceDao(com.webank.wedatasphere.qualitis.dao.TaskDataSourceDao) ExcelResult(com.webank.wedatasphere.qualitis.excel.ExcelResult) FileOutputStream(java.io.FileOutputStream) IOException(java.io.IOException) File(java.io.File) TaskRuleSimple(com.webank.wedatasphere.qualitis.entity.TaskRuleSimple) ExcelSheetName(com.webank.wedatasphere.qualitis.project.constant.ExcelSheetName) Paths(java.nio.file.Paths) ApplicationDao(com.webank.wedatasphere.qualitis.dao.ApplicationDao) UserDao(com.webank.wedatasphere.qualitis.dao.UserDao) TaskResultDao(com.webank.wedatasphere.qualitis.dao.TaskResultDao) TaskDataSource(com.webank.wedatasphere.qualitis.entity.TaskDataSource) Task(com.webank.wedatasphere.qualitis.entity.Task) User(com.webank.wedatasphere.qualitis.entity.User) ApplicationResponse(com.webank.wedatasphere.qualitis.response.ApplicationResponse) ArrayList(java.util.ArrayList) GeneralResponse(com.webank.wedatasphere.qualitis.response.GeneralResponse) TaskDataSource(com.webank.wedatasphere.qualitis.entity.TaskDataSource) GetAllResponse(com.webank.wedatasphere.qualitis.response.GetAllResponse) Application(com.webank.wedatasphere.qualitis.entity.Application)

Example 15 with Application

use of com.webank.wedatasphere.qualitis.entity.Application in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method commonExecution.

@Override
public ApplicationTaskSimpleResponse commonExecution(List<Rule> rules, StringBuffer partition, String executionUser, String nodeName, String startupParam, String clusterName, String setFlag, Map<String, String> execParams, Application newApplication, Date date, StringBuffer runDate) throws RuleVariableNotSupportException, JobSubmitException, RuleVariableNotFoundException, ArgumentException, ConvertException, DataQualityTaskException, TaskTypeException, ClusterInfoNotConfigException, SystemConfigException, UnExpectedRequestException, MetaDataAcquireFailedException, IOException, TaskNotExistException, SemanticException, ParseException, DataSourceMoveException, DataSourceOverSizeException, org.apache.hadoop.hive.ql.parse.ParseException, BothNullDatasourceException, RightNullDatasourceException, LeftNullDatasourceException, java.text.ParseException {
    // current user
    String userName = executionUser;
    // Generate database name.
    String database = generateDatabase(userName);
    LOGGER.info("Succeed to generate database_name: {}", database);
    // Save application
    newApplication.setRuleSize(rules.size());
    newApplication.setSavedDb(database);
    Application saveApplication = applicationDao.saveApplication(newApplication);
    List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
    List<Rule> fileRules = new ArrayList<>();
    Map<Long, Map> dataSourceMysqlConnect = new HashMap<>(2);
    for (Iterator<Rule> iterator = rules.iterator(); iterator.hasNext(); ) {
        Rule currentRule = iterator.next();
        if (currentRule.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
            // Replace with execution parameter and parse datasource to save.
            customReSaveDateSource(currentRule, execParams, clusterName, date);
        }
        List<Map<String, String>> mappingCols = new ArrayList<>();
        getMappingCols(currentRule, mappingCols);
        // Check datasource before submit job.
        try {
            checkDatasource(currentRule, userName, partition, mappingCols, nodeName, clusterName, dataSourceMysqlConnect);
        } catch (BothNullDatasourceException e) {
            Task taskInDb = taskDao.save(new Task(newApplication, newApplication.getSubmitTime(), TaskStatusEnum.SUCCEED.getCode()));
            taskInDb.setClusterName(clusterName);
            TaskRuleSimple taskRuleSimple = new TaskRuleSimple(currentRule, taskInDb);
            Set<TaskDataSource> taskDataSources = new HashSet<>(fileRules.size());
            Set<TaskRuleSimple> taskRuleSimples = new HashSet<>(fileRules.size());
            taskRuleSimples.add(taskRuleSimpleRepository.save(taskRuleSimple));
            for (RuleDataSource ruleDataSource : currentRule.getRuleDataSources()) {
                taskDataSources.add(taskDataSourceRepository.save(new TaskDataSource(ruleDataSource, taskInDb)));
            }
            taskInDb.setEndTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
            taskInDb.setTaskDataSources(taskDataSources);
            taskInDb.setTaskRuleSimples(taskRuleSimples);
            TaskResult taskResult = new TaskResult();
            taskResult.setApplicationId(newApplication.getId());
            taskResult.setCreateTime(newApplication.getSubmitTime());
            taskResult.setRuleId(currentRule.getId());
            taskResult.setResultType("Long");
            taskResult.setValue(0 + "");
            taskResultDao.saveTaskResult(taskResult);
            taskDao.save(taskInDb);
            iterator.remove();
            if (!iterator.hasNext()) {
                throw e;
            }
            continue;
        }
        if (currentRule.getRuleType().equals(RuleTypeEnum.FILE_TEMPLATE_RULE.getCode())) {
            fileRules.add(currentRule);
            LOGGER.info("Succeed to find file rule. Rule: {}", currentRule.getId() + " " + currentRule.getName());
            iterator.remove();
        }
    }
    String submitTime = ExecutionManagerImpl.PRINT_TIME_FORMAT.format(date);
    // General task.
    if (!rules.isEmpty()) {
        taskSubmitResults.addAll(executionManager.submitApplication(rules, nodeName, submitTime, userName, database, partition, date, saveApplication, clusterName, startupParam, setFlag, execParams, runDate, dataSourceMysqlConnect));
    }
    // Execute file rule task and save task result.
    if (!fileRules.isEmpty()) {
        taskSubmitResults.add(executionManager.executeFileRule(fileRules, submitTime, saveApplication, userName, clusterName, runDate));
    }
    saveApplication.setTotalTaskNum(taskSubmitResults.size());
    LOGGER.info("Succeed to submit application. result: {}", taskSubmitResults);
    Application applicationInDb = applicationDao.saveApplication(saveApplication);
    LOGGER.info("Succeed to save application. application: {}", applicationInDb);
    return new ApplicationTaskSimpleResponse(taskSubmitResults);
}
Also used : Task(com.webank.wedatasphere.qualitis.entity.Task) RuleDataSource(com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource) ApplicationTaskSimpleResponse(com.webank.wedatasphere.qualitis.response.ApplicationTaskSimpleResponse) Set(java.util.Set) HashSet(java.util.HashSet) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) Date(java.util.Date) TaskDataSource(com.webank.wedatasphere.qualitis.entity.TaskDataSource) BothNullDatasourceException(com.webank.wedatasphere.qualitis.exception.BothNullDatasourceException) TaskResult(com.webank.wedatasphere.qualitis.entity.TaskResult) Rule(com.webank.wedatasphere.qualitis.rule.entity.Rule) Application(com.webank.wedatasphere.qualitis.entity.Application) Map(java.util.Map) LinkedHashMap(java.util.LinkedHashMap) HashMap(java.util.HashMap) TaskRuleSimple(com.webank.wedatasphere.qualitis.entity.TaskRuleSimple) TaskSubmitResult(com.webank.wedatasphere.qualitis.bean.TaskSubmitResult)

Aggregations

Application (com.webank.wedatasphere.qualitis.entity.Application)22 Task (com.webank.wedatasphere.qualitis.entity.Task)15 GeneralResponse (com.webank.wedatasphere.qualitis.response.GeneralResponse)14 ArrayList (java.util.ArrayList)13 UnExpectedRequestException (com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException)10 Date (java.util.Date)9 TaskRuleSimple (com.webank.wedatasphere.qualitis.entity.TaskRuleSimple)7 HashMap (java.util.HashMap)7 TaskDataSource (com.webank.wedatasphere.qualitis.entity.TaskDataSource)6 TaskRuleAlarmConfig (com.webank.wedatasphere.qualitis.entity.TaskRuleAlarmConfig)6 User (com.webank.wedatasphere.qualitis.entity.User)6 TaskSubmitResult (com.webank.wedatasphere.qualitis.bean.TaskSubmitResult)5 LinkisConfig (com.webank.wedatasphere.qualitis.config.LinkisConfig)5 AlarmConfigStatusEnum (com.webank.wedatasphere.qualitis.constant.AlarmConfigStatusEnum)5 TaskStatusEnum (com.webank.wedatasphere.qualitis.constant.TaskStatusEnum)5 ApplicationDao (com.webank.wedatasphere.qualitis.dao.ApplicationDao)5 ClusterInfoDao (com.webank.wedatasphere.qualitis.dao.ClusterInfoDao)5 TaskDao (com.webank.wedatasphere.qualitis.dao.TaskDao)5 TaskResultDao (com.webank.wedatasphere.qualitis.dao.TaskResultDao)5 ClusterInfo (com.webank.wedatasphere.qualitis.entity.ClusterInfo)5