Examples with Project com.webank.wedatasphere.qualitis.project.entity.Project used on opensource projects

Example 26 with Project

use of com.webank.wedatasphere.qualitis.project.entity.Project in project Qualitis by WeBankFinTech.

the class ProjectBatchServiceImpl method getExcelRuleByProject.

private List<ExcelTemplateRuleByProject> getExcelRuleByProject(List<Project> projects, String localeStr) {
    List<ExcelTemplateRuleByProject> excelTemplateRuleByProjects = new ArrayList<>();
    for (Project project : projects) {
        List<Rule> rules = ruleDao.findByProject(project);
        List<ExcelTemplateRuleByProject> excelTemplateRules = ruleBatchService.getTemplateRule(rules, localeStr);
        for (ExcelTemplateRuleByProject templateRuleByProject : excelTemplateRules) {
            ExcelTemplateRuleByProject excelTemplateRuleByProject = new ExcelTemplateRuleByProject();
            BeanUtils.copyProperties(templateRuleByProject, excelTemplateRuleByProject);
            excelTemplateRuleByProject.setProjectName(project.getName());
            LOGGER.info("Collect excel line of template rule: {}", excelTemplateRuleByProject);
            excelTemplateRuleByProjects.add(excelTemplateRuleByProject);
        }
    }
    return excelTemplateRuleByProjects;
}

Example 27 with Project

use of com.webank.wedatasphere.qualitis.project.entity.Project in project Qualitis by WeBankFinTech.

the class RuleQueryServiceImpl method deleteRules.

@Override
@Transactional(rollbackFor = { RuntimeException.class, UnExpectedRequestException.class })
public void deleteRules(RulesDeleteRequest request) throws UnExpectedRequestException, PermissionDeniedRequestException {
    for (long ruleId : request.getRuleIdList()) {
        Rule ruleInDb = ruleDao.findById(ruleId);
        String loginUser = HttpUtils.getUserName(httpServletRequest);
        // Check existence of rule
        if (ruleInDb == null) {
            throw new UnExpectedRequestException("Rule id [" + ruleId + "]) {&DOES_NOT_EXIST}");
        }
        // Check existence of project
        Project projectInDb = projectService.checkProjectExistence(ruleInDb.getProject().getId(), loginUser);
        // Check permissions of project
        List<Integer> permissions = new ArrayList<>();
        permissions.add(ProjectUserPermissionEnum.DEVELOPER.getCode());
        projectService.checkProjectPermission(projectInDb, loginUser, permissions);
        // Update rule count of datasource
        ruleDataSourceService.updateRuleDataSourceCount(ruleInDb, -1);
        // Delete rule
        ruleDao.deleteRule(ruleInDb);
        LOGGER.info("Succeed to delete rule. rule_id: {}", ruleInDb.getId());
        if (ruleInDb.getRuleType().equals(RuleTypeEnum.CUSTOM_RULE.getCode())) {
            // Delete template of custom rule
            ruleTemplateService.deleteCustomTemplate(ruleInDb.getTemplate());
            LOGGER.info("Succeed to delete custom rule. rule_id: {}", ruleInDb.getId());
        }
    }
}

Example 28 with Project

use of com.webank.wedatasphere.qualitis.project.entity.Project in project Qualitis by WeBankFinTech.

the class MetaDataServiceImpl method saveRuleFailedTableInLabel.

private void saveRuleFailedTableInLabel(Set<String> failedTable, Long projectId) {
    if (failedTable.size() > 0) {
        Project projectInDb = projectDao.findById(projectId);
        projectService.addProjectLabels(failedTable, projectInDb);
        projectDao.saveProject(projectInDb);
    }
}

Example 29 with Project

use of com.webank.wedatasphere.qualitis.project.entity.Project in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method groupExecution.

@Override
public GeneralResponse<?> groupExecution(GroupExecutionRequest request, Integer invokeCode) throws UnExpectedRequestException, PermissionDeniedRequestException {
    LOGGER.info("Execute application by group. group_id: {}", request.getGroupId());
    LOGGER.info("Execute application by nodename. nodename: {}", request.getNodeName());
    // Check Arguments
    GroupExecutionRequest.checkRequest(request);
    String loginUser = getLoginUser(httpServletRequest, request.getCreateUser(), request.getAsync());
    // Check existence of project
    RuleGroup ruleGroupInDb = ruleGroupDao.findById(request.getGroupId());
    if (null == ruleGroupInDb) {
        throw new UnExpectedRequestException("Group_id " + request.getGroupId() + " {&DOES_NOT_EXIST}");
    }
    LOGGER.info("Succeed to find rule group. group_id: {}", ruleGroupInDb.getId());
    Project projectInDb = projectDao.findById(ruleGroupInDb.getProjectId());
    // Check permissions of project
    List<Integer> permissions = new ArrayList<>();
    permissions.add(ProjectUserPermissionEnum.OPERATOR.getCode());
    projectService.checkProjectPermission(projectInDb, loginUser, permissions);
    checkPermissionCreateUserProxyExecuteUser(request.getCreateUser(), request.getExecutionUser());
    // Find all rules
    List<Rule> rules = ruleDao.findByRuleGroup(ruleGroupInDb);
    if (CollectionUtils.isEmpty(rules)) {
        throw new UnExpectedRequestException("{&NO_RULE_CAN_BE_EXECUTED}");
    }
    List<Long> ruleIds = rules.stream().map(Rule::getId).collect(Collectors.toList());
    StringBuffer runDate = new StringBuffer();
    StringBuffer partition = new StringBuffer();
    Map<String, String> execParamMap = new HashMap<>(5);
    parseExecParams(partition, runDate, request.getExecutionParam(), execParamMap);
    List<ApplicationSubmitRequest> applicationSubmitRequests = new ArrayList<>();
    // Construct dynamic partition.
    try {
        submitRulesWithDynamicPartition(applicationSubmitRequests, projectInDb.getId(), ruleGroupInDb.getId(), rules, ruleIds, request.getExecutionUser(), request.getDyNamicPartition(), request.getClusterName(), partition, request.getDyNamicPartitionPrefix());
    } catch (ResourceAccessException e) {
        generateAbnormalApplicationInfo(projectInDb.getId(), request.getGroupId(), request.getCreateUser(), request.getExecutionUser(), new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
        LOGGER.error("One group execution[id={}] of the project execution start failed!", request.getGroupId());
    } catch (NoPartitionException e) {
        generateAbnormalApplicationInfo(projectInDb.getId(), request.getGroupId(), request.getCreateUser(), request.getExecutionUser(), new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
        LOGGER.error("One group execution[id={}] of the project execution start failed!", request.getGroupId());
    } catch (Exception e) {
        generateAbnormalApplicationInfo(projectInDb.getId(), request.getGroupId(), request.getCreateUser(), request.getExecutionUser(), new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.UNKNOWN_ERROR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
        LOGGER.error("One group execution[id={}] of the project execution start failed!", request.getGroupId());
    }
    if (CollectionUtils.isEmpty(ruleIds)) {
        return new GeneralResponse<>("200", "{&SUCCEED_TO_DISPATCH_TASK}", null);
    }
    GeneralResponse<ApplicationTaskSimpleResponse> generalResponse = (GeneralResponse<ApplicationTaskSimpleResponse>) outerExecutionService.submitRules(ruleIds, partition, loginUser, request.getExecutionUser(), request.getNodeName(), projectInDb.getId(), ruleGroupInDb.getId(), request.getStartupParamName(), request.getClusterName(), request.getSetFlag(), execParamMap, request.getExecutionParam(), runDate, invokeCode);
    // Record project event.
    projectEventService.record(projectInDb.getId(), loginUser, "submit group execution", "group[name=" + ruleGroupInDb.getRuleGroupName() + "].", EventTypeEnum.SUBMIT_PROJECT.getCode());
    return generalResponse;
}

Example 30 with Project

use of com.webank.wedatasphere.qualitis.project.entity.Project in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method projectExecution.

@Override
@Transactional(rollbackFor = { RuntimeException.class, UnExpectedRequestException.class })
public GeneralResponse<?> projectExecution(ProjectExecutionRequest request, Integer invokeCode) throws UnExpectedRequestException, PermissionDeniedRequestException {
    LOGGER.info("Execute application by project. project_id: {}", request.getProjectId());
    // Check Arguments
    ProjectExecutionRequest.checkRequest(request);
    String executionUser = request.getExecutionUser();
    String loginUser = getLoginUser(httpServletRequest, request.getCreateUser(), request.getAsync());
    LOGGER.info("Qualitis execution user: {}", executionUser);
    LOGGER.info("Qualitis login or create user: {}", loginUser);
    // Check existence of project
    Project projectInDb = projectDao.findById(request.getProjectId());
    if (null == projectInDb) {
        throw new UnExpectedRequestException("Project_id " + request.getProjectId() + " {&DOES_NOT_EXIST}");
    }
    // Check permissions of project
    List<Integer> permissions = new ArrayList<>();
    permissions.add(ProjectUserPermissionEnum.OPERATOR.getCode());
    projectService.checkProjectPermission(projectInDb, loginUser, permissions);
    checkPermissionCreateUserProxyExecuteUser(request.getCreateUser(), request.getExecutionUser());
    // Find all rule group
    List<RuleGroup> ruleGroups = ruleGroupDao.findByProjectId(projectInDb.getId());
    if (CollectionUtils.isEmpty(ruleGroups)) {
        throw new UnExpectedRequestException("{&NO_RULE_CAN_BE_EXECUTED}");
    }
    LOGGER.info("Succeed to find rule group list from project[id={}], rule group[{}]", projectInDb.getId(), Arrays.toString(ruleGroups.toArray()));
    // Parse partition and run date from execution parameters.
    StringBuffer partition = new StringBuffer();
    StringBuffer runDate = new StringBuffer();
    Map<String, String> execParamMap = new HashMap<>(5);
    parseExecParams(partition, runDate, request.getExecutionParam(), execParamMap);
    ApplicationProjectResponse applicationProjectResponse = new ApplicationProjectResponse();
    List<ApplicationSubmitRequest> applicationSubmitRequests = new ArrayList<>(ruleGroups.size());
    for (RuleGroup ruleGroup : ruleGroups) {
        // Find all rules
        List<Rule> rules = ruleDao.findByRuleGroup(ruleGroup);
        if (CollectionUtils.isEmpty(rules)) {
            ruleGroupDao.delete(ruleGroup);
            continue;
        }
        LOGGER.info("Succeed to get rules from rule group[id={}].", ruleGroup.getId());
        List<Long> ruleIds = rules.stream().map(Rule::getId).collect(Collectors.toList());
        try {
            // Dynamic partition.
            submitRulesWithDynamicPartition(applicationSubmitRequests, projectInDb.getId(), ruleGroup.getId(), rules, ruleIds, executionUser, request.getDyNamicPartition(), request.getClusterName(), partition, request.getDyNamicPartitionPrefix());
            if (CollectionUtils.isEmpty(ruleIds)) {
                continue;
            }
        } catch (ResourceAccessException e) {
            generateAbnormalApplicationInfo(request.getProjectId(), ruleGroup.getId(), request.getCreateUser(), executionUser, new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
            LOGGER.error("One group execution[id={}] of the project execution start failed!", ruleGroup.getId());
        } catch (NoPartitionException e) {
            generateAbnormalApplicationInfo(request.getProjectId(), ruleGroup.getId(), request.getCreateUser(), executionUser, new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.METADATA_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
            LOGGER.error("One group execution[id={}] of the project execution start failed!", ruleGroup.getId());
        } catch (Exception e) {
            generateAbnormalApplicationInfo(request.getProjectId(), ruleGroup.getId(), request.getCreateUser(), executionUser, new Date(), invokeCode, partition.toString(), request.getStartupParamName(), request.getExecutionParam(), e, ApplicationCommentEnum.UNKNOWN_ERROR_ISSUES.getCode(), ApplicationStatusEnum.TASK_SUBMIT_FAILED.getCode(), rules);
            LOGGER.error("One group execution[id={}] of the project execution start failed!", ruleGroup.getId());
        }
        applicationSubmitRequests.add(new ApplicationSubmitRequest(projectInDb.getId(), ruleGroup.getId(), ruleIds, partition));
    }
    for (ApplicationSubmitRequest applicationSubmitRequest : applicationSubmitRequests) {
        GeneralResponse<?> generalResponse = outerExecutionService.submitRules(applicationSubmitRequest.getRuleIds(), applicationSubmitRequest.getPartition(), loginUser, executionUser, DEFAULT_NODE_NAME, projectInDb.getId(), applicationSubmitRequest.getRuleGroupId(), request.getStartupParamName(), request.getClusterName(), request.getSetFlag(), execParamMap, request.getExecutionParam(), runDate, invokeCode);
        applicationProjectResponse.getApplicationTaskSimpleResponses().add((ApplicationTaskSimpleResponse) generalResponse.getData());
    }
    // Record project event.
    projectEventService.record(projectInDb.getId(), loginUser, "submit project execution", ".", EventTypeEnum.SUBMIT_PROJECT.getCode());
    return new GeneralResponse<>("200", "{&SUCCEED_TO_DISPATCH_TASK}", applicationProjectResponse);
}