Examples with Project com.webank.wedatasphere.qualitis.project.entity.Project used on opensource projects

Example 31 with Project

use of com.webank.wedatasphere.qualitis.project.entity.Project in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method dataSourceExecution.

@Override
public GeneralResponse<?> dataSourceExecution(DataSourceExecutionRequest request) throws UnExpectedRequestException, PermissionDeniedRequestException {
    LOGGER.info("Execute application by datasource. cluster: {}, database: {}, table: {}", request.getCluster(), request.getDatabase(), request.getTable());
    // Check Arguments.
    DataSourceExecutionRequest.checkRequest(request);
    String loginUser = getLoginUser(httpServletRequest, request.getCreateUser(), request.getAsync());
    // Find all rule datasources by user.
    List<RuleDataSource> ruleDataSources = new ArrayList<>();
    ruleDataSources.addAll(ruleDataSourceDao.findDatasourcesByUser(loginUser, request.getCluster(), request.getDatabase(), request.getTable()));
    List<Rule> rules = ruleDataSources.stream().filter(r -> r.getClusterName().equals(request.getCluster()) && r.getDbName().equals(request.getDatabase()) && r.getTableName().equals(request.getTable())).map(RuleDataSource::getRule).distinct().filter(rule -> {
        if (!request.getCrossTable()) {
            return rule.getRuleDataSources().size() == 1;
        }
        return true;
    }).collect(Collectors.toList());
    if (CollectionUtils.isEmpty(rules)) {
        throw new UnExpectedRequestException("{&NO_RULE_CAN_BE_EXECUTED}");
    }
    checkPermissionCreateUserProxyExecuteUser(request.getCreateUser(), request.getExecutionUser());
    ApplicationProjectResponse applicationProjectResponse = new ApplicationProjectResponse();
    List<ApplicationSubmitRequest> applicationSubmitRequests = new ArrayList<>();
    StringBuffer partition = new StringBuffer();
    StringBuffer runDate = new StringBuffer();
    Map<String, String> execParamMap = new HashMap<>(5);
    parseExecParams(partition, runDate, request.getExecutionParam(), execParamMap);
    List<Project> projects = rules.stream().map(Rule::getProject).distinct().collect(Collectors.toList());
    for (Project projectInDb : projects) {
        // Check permissions of project
        List<Integer> permissions = new ArrayList<>();
        permissions.add(ProjectUserPermissionEnum.OPERATOR.getCode());
        projectService.checkProjectPermission(projectInDb, loginUser, permissions);
        List<Rule> currentRules = rules.stream().filter(rule -> rule.getProject().getId().equals(projectInDb.getId())).collect(Collectors.toList());
        List<RuleGroup> currentRuleGroups = currentRules.stream().map(Rule::getRuleGroup).distinct().collect(Collectors.toList());
        for (RuleGroup ruleGroup : currentRuleGroups) {
            List<Rule> currentRulesOfGroup = currentRules.stream().filter(rule -> rule.getRuleGroup().getRuleGroupName().equals(ruleGroup.getRuleGroupName())).distinct().collect(Collectors.toList());
            List<Long> currentRuleIds = currentRulesOfGroup.stream().map(Rule::getId).distinct().collect(Collectors.toList());
            LOGGER.info("Succeed to find current rules of one group with datasource. rule_id: {}", currentRuleIds);
            // Dynamic partition.
            try {
                submitRulesWithDynamicPartition(applicationSubmitRequests, projectInDb.getId(), ruleGroup.getId(), rules, currentRuleIds, request.getExecutionUser(), request.getDyNamicPartition(), request.getClusterName(), partition, request.getDyNamicPartitionPrefix());
            } catch (ResourceAccessException e) {
                // Record submit failed applicatoin.
                generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
                LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
            } catch (NoPartitionException e) {
                // Record submit failed applicatoin.
                generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
                LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
            } catch (Exception e) {
                // Record submit failed applicatoin.
                generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode(), partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.UNKNOWN_ERROR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
                LOGGER.error("One group execution[id={}] of the datasource execution start failed!", ruleGroup.getId());
            }
            if (CollectionUtils.isNotEmpty(currentRuleIds)) {
                applicationSubmitRequests.add(new ApplicationSubmitRequest(request.getJobId(), projectInDb.getId(), ruleGroup.getId(), currentRuleIds, partition));
            }
        }
        projectEventService.record(projectInDb.getId(), loginUser, "submit datasource execution", "rule name[" + Arrays.toString(rules.stream().filter(rule -> rule.getProject().getId().equals(projectInDb.getId())).toArray()) + "].", EventTypeEnum.SUBMIT_PROJECT.getCode());
    }
    GeneralResponse<ApplicationTaskSimpleResponse> generalResponse;
    for (ApplicationSubmitRequest applicationSubmitRequest : applicationSubmitRequests) {
        if (StringUtils.isNotBlank(request.getJobId())) {
            applicationSubmitRequest.setJobId(request.getJobId());
        }
        generalResponse = (GeneralResponse<ApplicationTaskSimpleResponse>) outerExecutionService.submitRules(applicationSubmitRequest.getRuleIds(), applicationSubmitRequest.getPartition(), loginUser, request.getExecutionUser(), DEFAULT_NODE_NAME, applicationSubmitRequest.getProjectId(), applicationSubmitRequest.getRuleGroupId(), request.getStartupParamName(), request.getClusterName(), request.getSetFlag(), execParamMap, request.getExecutionParam(), runDate, InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode());
        applicationProjectResponse.getApplicationTaskSimpleResponses().add(generalResponse.getData());
    }
    return new GeneralResponse<>("200", "{&SUCCEED_TO_DISPATCH_TASK}", applicationProjectResponse);
}

Example 32 with Project

use of com.webank.wedatasphere.qualitis.project.entity.Project in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method ruleListExecution.

@Override
public GeneralResponse<?> ruleListExecution(RuleListExecutionRequest request, Integer invokeCode) throws UnExpectedRequestException, PermissionDeniedRequestException {
    LOGGER.info("Execute application by rules. rule_ids: {}", request.getRuleList());
    // Check Arguments.
    RuleListExecutionRequest.checkRequest(request);
    String loginUser = getLoginUser(httpServletRequest, request.getCreateUser(), request.getAsync());
    // Check the existence of rule.
    List<Rule> rules = ruleDao.findByIds(request.getRuleList());
    if (CollectionUtils.isEmpty(rules)) {
        throw new UnExpectedRequestException("{&NO_RULE_CAN_BE_EXECUTED}");
    }
    List<Long> ruleIds = rules.stream().map(Rule::getId).distinct().collect(Collectors.toList());
    // Check rule exists on the request.
    List<Long> notExistRules = request.getRuleList().stream().filter(ruleId -> !ruleIds.contains(ruleId)).collect(Collectors.toList());
    if (!notExistRules.isEmpty()) {
        throw new UnExpectedRequestException("The ids of rule: " + notExistRules.toString() + " {&DOES_NOT_EXIST}");
    }
    LOGGER.info("Succeed to find all rules.");
    checkPermissionCreateUserProxyExecuteUser(request.getCreateUser(), request.getExecutionUser());
    StringBuffer runDate = new StringBuffer();
    StringBuffer partition = new StringBuffer();
    Map<String, String> execParamMap = new HashMap<>(5);
    parseExecParams(partition, runDate, request.getExecutionParam(), execParamMap);
    ApplicationProjectResponse applicationProjectResponse = new ApplicationProjectResponse();
    List<ApplicationSubmitRequest> applicationSubmitRequests = new ArrayList<>(rules.size());
    List<Project> projects = rules.stream().map(Rule::getProject).distinct().collect(Collectors.toList());
    GeneralResponse<ApplicationTaskSimpleResponse> generalResponse;
    // Check permissions of projects.
    for (Project projectInDb : projects) {
        List<Integer> permissions = new ArrayList<>();
        permissions.add(ProjectUserPermissionEnum.OPERATOR.getCode());
        projectService.checkProjectPermission(projectInDb, loginUser, permissions);
        List<Rule> currentRules = rules.stream().filter(rule -> rule.getProject().getId().equals(projectInDb.getId())).collect(Collectors.toList());
        List<RuleGroup> currentRuleGroups = currentRules.stream().map(Rule::getRuleGroup).distinct().collect(Collectors.toList());
        for (RuleGroup ruleGroup : currentRuleGroups) {
            List<Rule> currentRulesOfGroup = currentRules.stream().filter(rule -> rule.getRuleGroup().getRuleGroupName().equals(ruleGroup.getRuleGroupName())).distinct().collect(Collectors.toList());
            List<Long> currentRuleIds = currentRulesOfGroup.stream().map(Rule::getId).distinct().collect(Collectors.toList());
            // Dynamic partition.
            try {
                submitRulesWithDynamicPartition(applicationSubmitRequests, projectInDb.getId(), ruleGroup.getId(), currentRulesOfGroup, currentRuleIds, request.getExecutionUser(), request.getDyNamicPartition(), request.getClusterName(), partition, request.getDyNamicPartitionPrefix());
                if (CollectionUtils.isEmpty(currentRuleIds)) {
                    continue;
                }
            } catch (ResourceAccessException e) {
                // Record submit failed applicatoin.
                generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
                LOGGER.error("One group execution[id={}] of the rule list execution start failed!", ruleGroup.getId());
            } catch (NoPartitionException e) {
                // Record submit failed applicatoin.
                generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
                LOGGER.error("One group execution[id={}] of the rule list execution start failed!", ruleGroup.getId());
            } catch (Exception e) {
                // Record submit failed applicatoin.
                generateAbnormalApplicationInfo(projectInDb.getId(), ruleGroup.getId(), request.getCreateUser(), request.getExecutionUser(), new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
                LOGGER.error("One group execution[id={}] of the rule list execution start failed!", ruleGroup.getId());
            }
            applicationSubmitRequests.add(new ApplicationSubmitRequest(projectInDb.getId(), ruleGroup.getId(), currentRuleIds, partition));
        }
        projectEventService.record(projectInDb.getId(), loginUser, "submit rule list execution", "rule name[" + Arrays.toString(currentRules.toArray()) + "].", EventTypeEnum.SUBMIT_PROJECT.getCode());
    }
    for (ApplicationSubmitRequest applicationSubmitRequest : applicationSubmitRequests) {
        generalResponse = (GeneralResponse<ApplicationTaskSimpleResponse>) outerExecutionService.submitRules(applicationSubmitRequest.getRuleIds(), applicationSubmitRequest.getPartition(), loginUser, request.getExecutionUser(), DEFAULT_NODE_NAME, applicationSubmitRequest.getProjectId(), applicationSubmitRequest.getRuleGroupId(), request.getStartupParamName(), request.getClusterName(), request.getSetFlag(), execParamMap, request.getExecutionParam(), runDate, invokeCode);
        applicationProjectResponse.getApplicationTaskSimpleResponses().add(generalResponse.getData());
    }
    return new GeneralResponse<>("200", "{&SUCCEED_TO_DISPATCH_TASK}", applicationProjectResponse);
}

Example 33 with Project

use of com.webank.wedatasphere.qualitis.project.entity.Project in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method catchAndSolve.

private void catchAndSolve(Exception e, Integer commentCode, Integer statusCode, List<Rule> rules, Application newApplication) {
    LOGGER.error(e.getMessage(), e);
    newApplication.setApplicationComment(commentCode);
    newApplication.setStatus(statusCode);
    // Record rules info and datasource info.
    StringBuffer info = new StringBuffer();
    newApplication.setFinishTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
    for (Rule rule : rules) {
        info.append("Rule[").append(rule.getName()).append("; ");
        info.append("datasource: ").append(rule.getRuleDataSources().stream().filter(ruleDataSource -> StringUtils.isNotBlank(ruleDataSource.getDbName()) && StringUtils.isNotBlank(ruleDataSource.getTableName())).map(ruleDataSource -> ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + " ").collect(Collectors.joining()));
    }
    newApplication.setExceptionMessage(info.append("]\n").append(ExceptionUtils.getStackTrace(e)).toString());
    if (CollectionUtils.isNotEmpty(rules)) {
        Project currentProject = rules.iterator().next().getProject();
        newApplication.setProjectName(StringUtils.isNotBlank(currentProject.getCnName()) ? currentProject.getCnName() : currentProject.getName());
        newApplication.setRuleGroupName(rules.iterator().next().getRuleGroup().getRuleGroupName());
        newApplication.setRuleDatesource(rules.stream().map(Rule::getRuleDataSources).flatMap(ruleDataSources -> ruleDataSources.stream()).filter(ruleDataSource -> StringUtils.isNotBlank(ruleDataSource.getDbName()) && StringUtils.isNotBlank(ruleDataSource.getTableName())).map(ruleDataSource -> ruleDataSource.getDbName() + "." + ruleDataSource.getTableName() + " ").collect(Collectors.joining()));
    }
    applicationDao.saveApplication(newApplication);
    LOGGER.info("Succeed to set application status to [{}], application_id: [{}]", newApplication.getStatus(), newApplication.getId());
}

Example 34 with Project

use of com.webank.wedatasphere.qualitis.project.entity.Project in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method generalExecution.

@Override
@Transactional(rollbackFor = { RuntimeException.class, UnExpectedRequestException.class })
public GeneralResponse<?> generalExecution(GeneralExecutionRequest request) throws UnExpectedRequestException, PermissionDeniedRequestException {
    // Generator application information
    GeneralExecutionRequest.checkRequest(request);
    if (request.getDyNamicPartitionPrefix() == null) {
        request.setDyNamicPartitionPrefix("");
    }
    if (request.getProjectId() != null) {
        ProjectExecutionRequest projectExecutionRequest = new ProjectExecutionRequest();
        BeanUtils.copyProperties(request, projectExecutionRequest);
        return projectExecution(projectExecutionRequest, InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode());
    } else if (CollectionUtils.isNotEmpty(request.getRuleList())) {
        RuleListExecutionRequest ruleListExecutionRequest = new RuleListExecutionRequest();
        BeanUtils.copyProperties(request, ruleListExecutionRequest);
        return ruleListExecution(ruleListExecutionRequest, InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode());
    } else if (request.getGroupId() != null) {
        GroupExecutionRequest groupExecutionRequest = new GroupExecutionRequest();
        BeanUtils.copyProperties(request, groupExecutionRequest);
        return groupExecution(groupExecutionRequest, InvokeTypeEnum.FLOW_API_INVOKE.getCode());
    } else if (StringUtils.isNotBlank(request.getProjectName())) {
        Project projectInDb = projectDao.findByNameAndCreateUser(request.getProjectName(), request.getCreateUser());
        if (projectInDb == null) {
            List<Integer> permissions = new ArrayList<>();
            permissions.add(ProjectUserPermissionEnum.CREATOR.getCode());
            permissions.add(ProjectUserPermissionEnum.DEVELOPER.getCode());
            List<Project> projects = projectUserDao.findByUsernameAndPermission(request.getExecutionUser(), permissions);
            if (CollectionUtils.isEmpty(projects)) {
                throw new UnExpectedRequestException("Pick project has no permissions.");
            }
            List<Project> execProject = projects.stream().filter(project -> project.getName().equals(request.getProjectName())).collect(Collectors.toList());
            if (CollectionUtils.isEmpty(execProject)) {
                throw new UnExpectedRequestException("Pick project has no permissions.");
            } else if (execProject.size() > 1) {
                throw new UnExpectedRequestException("Pick project has dubplicate names.");
            }
            projectInDb = execProject.iterator().next();
        }
        if (CollectionUtils.isNotEmpty(request.getRuleNameList())) {
            List<Long> ruleIds = new ArrayList<>(request.getRuleNameList().size());
            for (String ruleName : request.getRuleNameList()) {
                Rule rule = ruleDao.findByProjectAndRuleName(projectInDb, ruleName);
                ruleIds.add(rule.getId());
            }
            request.setRuleList(ruleIds);
            RuleListExecutionRequest ruleListExecutionRequest = new RuleListExecutionRequest();
            BeanUtils.copyProperties(request, ruleListExecutionRequest);
            return ruleListExecution(ruleListExecutionRequest, InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode());
        } else {
            request.setProjectId(projectInDb.getId());
            ProjectExecutionRequest projectExecutionRequest = new ProjectExecutionRequest();
            BeanUtils.copyProperties(request, projectExecutionRequest);
            return projectExecution(projectExecutionRequest, InvokeTypeEnum.BDP_CLIENT_API_INVOKE.getCode());
        }
    } else if (StringUtils.isNotBlank(request.getCluster())) {
        DataSourceExecutionRequest dataSourceExecutionRequest = new DataSourceExecutionRequest();
        BeanUtils.copyProperties(request, dataSourceExecutionRequest);
        return dataSourceExecution(dataSourceExecutionRequest);
    } else {
        LOGGER.error("Can not resolve the request: " + request);
        throw new UnExpectedRequestException("{&CAN_NOT_RESOLVE_THE_REQUEST}");
    }
}

Example 35 with Project

use of com.webank.wedatasphere.qualitis.project.entity.Project in project Qualitis by WeBankFinTech.

the class MultiSourceRuleServiceImpl method addMultiSourceRuleReal.

private GeneralResponse<RuleResponse> addMultiSourceRuleReal(AddMultiSourceRuleRequest request, boolean check) throws UnExpectedRequestException, PermissionDeniedRequestException {
    // Check Arguments
    CommonChecker.checkObject(request, "Request");
    String csId = request.getCsId();
    boolean cs = false;
    if (StringUtils.isNotBlank(csId)) {
        cs = true;
    }
    AddMultiSourceRuleRequest.checkRequest(request, false, cs);
    String loginUser = "";
    if (request.getLoginUser() != null) {
        loginUser = request.getLoginUser();
        LOGGER.info("Recover user[{}] from is adding rule.", loginUser);
    } else {
        loginUser = HttpUtils.getUserName(httpServletRequest);
    }
    LOGGER.info("Check permission user[{}] who is adding rule.", loginUser);
    // Check existence of project
    Project projectInDb = projectService.checkProjectExistence(request.getProjectId(), loginUser);
    // Check permissions of project
    if (check) {
        List<Integer> permissions = new ArrayList<>();
        permissions.add(ProjectUserPermissionEnum.DEVELOPER.getCode());
        projectService.checkProjectPermission(projectInDb, loginUser, permissions);
    }
    // Check existence of rule name
    ruleService.checkRuleName(request.getRuleName(), projectInDb, null);
    // Check existence of cluster
    ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(request.getClusterName());
    if (clusterInfo == null) {
        throw new UnExpectedRequestException("Cluster :" + request.getClusterName() + " {&DOES_NOT_EXIST}");
    }
    // Check existence of id and check if multi-table rule
    Template templateInDb = ruleTemplateService.checkRuleTemplate(request.getMultiSourceRuleTemplateId());
    if (!templateInDb.getTemplateType().equals(RuleTemplateTypeEnum.MULTI_SOURCE_TEMPLATE.getCode())) {
        throw new UnExpectedRequestException("Template id :" + request.getMultiSourceRuleTemplateId() + " {&IS_NOT_A_MULTI_SOURCE_TEMPLATE}");
    }
    RuleGroup ruleGroup;
    if (request.getRuleGroupId() != null) {
        ruleGroup = ruleGroupDao.findById(request.getRuleGroupId());
        if (ruleGroup == null) {
            throw new UnExpectedRequestException(String.format("Rule Group: %s {&CAN_NOT_BE_NULL_OR_EMPTY}", request.getRuleGroupId()));
        }
    } else {
        ruleGroup = ruleGroupDao.saveRuleGroup(new RuleGroup("Group_" + UUID.randomUUID().toString().replace("-", ""), projectInDb.getId()));
    }
    String leftUuid = UuidGenerator.generate();
    String rightUuid = UuidGenerator.generate();
    Rule savedRule = generateRule(request, projectInDb, false, null, false, cs, leftUuid, rightUuid, loginUser);
    savedRule.setRuleGroup(ruleGroup);
    savedRule.setSpecifyStaticStartupParam(request.getSpecifyStaticStartupParam());
    if (request.getSpecifyStaticStartupParam() != null && request.getSpecifyStaticStartupParam()) {
        savedRule.setStaticStartupParam(request.getStaticStartupParam());
    }
    savedRule = ruleDao.saveRule(savedRule);
    if (templateInDb.getChildTemplate() != null) {
        // Generate child rule
        AddMultiSourceRuleRequest addMultiSourceRuleRequest = generateChildRequest(request, templateInDb.getChildTemplate());
        Rule childRule = generateRule(addMultiSourceRuleRequest, null, false, null, true, cs, leftUuid, rightUuid, loginUser);
        childRule.setParentRule(savedRule);
        ruleDao.saveRule(childRule);
    }
    RuleResponse response = new RuleResponse(savedRule);
    LOGGER.info("Succeed to add multi source rule, rule_id: {}", savedRule.getId());
    return new GeneralResponse<>("200", "{&ADD_MULTI_RULE_SUCCESSFULLY}", response);
}