Examples with UnExpectedRequestException com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException used on opensource projects

Example 16 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class RuleBatchServiceImpl method getFileAlarmConfig.

private void getFileAlarmConfig(List<FileAlarmConfigRequest> alarmConfigRequests, ExcelTemplateFileRuleByProject excelTemplateFileRule, String localeStr) throws UnExpectedRequestException {
    String templateOutputName = excelTemplateFileRule.getAlarmCheckName();
    if (!StringUtils.isBlank(templateOutputName)) {
        Integer unit = FileOutputUnitEnum.fileOutputUnitCode(excelTemplateFileRule.getUnit());
        String checkTemplateName = excelTemplateFileRule.getCheckTemplateName();
        String compareTypeName = excelTemplateFileRule.getCompareType();
        String threshold = excelTemplateFileRule.getThreshold();
        FileAlarmConfigRequest fileAlarmConfigRequest = new FileAlarmConfigRequest();
        fileAlarmConfigRequest.setFileOutputName(FileOutputNameEnum.getFileOutputNameCode(templateOutputName, localeStr));
        fileAlarmConfigRequest.setCheckTemplate(CheckTemplateEnum.getCheckTemplateCode(checkTemplateName, localeStr));
        fileAlarmConfigRequest.setCompareType(CompareTypeEnum.getCompareTypeCode(compareTypeName));
        fileAlarmConfigRequest.setThreshold(Double.valueOf(threshold));
        fileAlarmConfigRequest.setFileOutputUnit(unit);
        // Rule Metric.
        String ruleMetricEnCode = excelTemplateFileRule.getRuleMetricEnCode();
        if (StringUtils.isNotBlank(ruleMetricEnCode)) {
            RuleMetric ruleMetricInDb = ruleMetricDao.findByEnCode(ruleMetricEnCode);
            if (ruleMetricInDb == null) {
                throw new UnExpectedRequestException("Rule metric[Code=" + ruleMetricEnCode + "] " + "{&DOES_NOT_EXIST}");
            }
            String code = ruleMetricInDb.getEnCode();
            fileAlarmConfigRequest.setRuleMetricEnCode(code);
        }
        alarmConfigRequests.add(fileAlarmConfigRequest);
    }
}

Example 17 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class ProjectUserServiceImpl method getAllProjectUser.

@Override
@Transactional(propagation = Propagation.REQUIRED, readOnly = true, rollbackFor = { RuntimeException.class, UnExpectedRequestException.class })
public GeneralResponse<List<ProjectUserResponse>> getAllProjectUser(Long projectId) throws UnExpectedRequestException, PermissionDeniedRequestException {
    List<ProjectUserResponse> projectUserResponses = new ArrayList<>();
    Project projectInDb = projectDao.findById(projectId);
    if (projectInDb == null) {
        throw new UnExpectedRequestException("{&PROJECT}: [ID=" + projectId + "] {&DOES_NOT_EXIST}");
    }
    String userName = HttpUtils.getUserName(httpRequest);
    List<ProjectUser> projectUsers = projectUserDao.findByProject(projectInDb);
    List<String> projectUserNames = projectUsers.stream().map(ProjectUser::getUserName).collect(Collectors.toList());
    if (!projectUserNames.contains(userName)) {
        throw new PermissionDeniedRequestException("{&HAS_NO_PERMISSION_TO_ACCESS}", 403);
    }
    for (ProjectUser currentProjectUser : projectUsers) {
        List<String> userNames = projectUserResponses.stream().map(ProjectUserResponse::getAuthorizedUser).collect(Collectors.toList());
        String currentUser = currentProjectUser.getUserName();
        if (userNames.contains(currentUser)) {
            continue;
        }
        String projectCreator = projectInDb.getCreateUser();
        ProjectUserResponse projectUserResponse = new ProjectUserResponse(projectInDb.getName(), projectCreator, currentUser);
        List<Integer> permissions = getPermissionList(projectUsers, currentUser);
        projectUserResponse.setPermissions(permissions);
        projectUserResponses.add(projectUserResponse);
    }
    return new GeneralResponse<>("200", "{&SUCCESS_TO_GET_PROJECT_USER}", projectUserResponses);
}

Example 18 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class AddRuleRequestBuilder method solveDatasource.

private void solveDatasource(String datasource) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    List<DataSourceRequest> dataSourceRequests = new ArrayList<>(1);
    List<DataSourceColumnRequest> dataSourceColumnRequests = new ArrayList<>(1);
    DataSourceRequest dataSourceRequest = new DataSourceRequest();
    String clusterName;
    String database;
    String table;
    String col;
    String filter;
    String[] datasourceStrs = datasource.split(SpecCharEnum.COLON.getValue());
    if (datasourceStrs.length > TWO) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    String dbAndTable = datasourceStrs[0];
    String dataSourceId = "";
    Matcher matcherId = DATA_SOURCE_ID.matcher(dbAndTable.toUpperCase());
    String dataSourceName = "";
    Matcher matcherName = DATA_SOURCE_NAME.matcher(dbAndTable.toUpperCase());
    while (matcherId.find()) {
        String group = matcherId.group();
        dataSourceId = group.replace(".(", "").replace(")", "").split("=")[1];
        int startIndex = dbAndTable.toUpperCase().indexOf(group);
        String replaceStr = dbAndTable.substring(startIndex, startIndex + group.length());
        dbAndTable = dbAndTable.replace(replaceStr, "");
    }
    if (StringUtils.isBlank(dataSourceId)) {
        while (matcherName.find()) {
            String group = matcherName.group();
            int startIndex = dbAndTable.toUpperCase().indexOf(group);
            String replaceStr = dbAndTable.substring(startIndex, startIndex + group.length());
            dataSourceName = replaceStr.replace(".(", "").replace(")", "").split("=")[1];
            dbAndTable = dbAndTable.replace(replaceStr, "");
        }
    }
    String[] datasources = dbAndTable.split(SpecCharEnum.PERIOD.getValue());
    filter = datasourceStrs[1];
    if (datasources.length != FOUR && datasources.length != THREE) {
        throw new UnExpectedRequestException("Datasource param is illegle");
    }
    clusterName = datasources[0];
    database = datasources[1];
    table = datasources[2];
    col = datasources.length >= FOUR ? datasources[3] : "";
    List<ColumnInfoDetail> cols = metaDataClient.getColumnInfo(clusterName, database, table, userName);
    if (StringUtils.isNotBlank(dataSourceId)) {
        LOGGER.info("Find data source connect. Data source ID: " + dataSourceId);
        dataSourceRequest.setLinkisDataSourceId(Long.parseLong(dataSourceId));
        GeneralResponse<Map> response = metaDataClient.getDataSourceInfoDetail(clusterName, userName, dataSourceRequest.getLinkisDataSourceId(), null);
        cols = metaDataClient.getColumnsByDataSource(clusterName, userName, dataSourceRequest.getLinkisDataSourceId(), database, table).getContent();
        Map dataSourceInfo = ((Map) response.getData().get("info"));
        String dataSourceInfoName = (String) dataSourceInfo.get("dataSourceName");
        String dataSourceInfoType = (String) ((Map) dataSourceInfo.get("dataSourceType")).get("name");
        dataSourceRequest.setLinkisDataSourceName(dataSourceInfoName);
        dataSourceRequest.setLinkisDataSourceType(dataSourceInfoType);
    } else if (StringUtils.isNotBlank(dataSourceName)) {
        LOGGER.info("Find data source connect. Data source name: " + dataSourceName);
        GeneralResponse<Map> response = metaDataClient.getDataSourceInfoDetailByName(clusterName, userName, dataSourceName);
        Map dataSourceInfo = ((Map) response.getData().get("info"));
        String dataSourceInfoName = (String) dataSourceInfo.get("dataSourceName");
        String dataSourceInfoType = (String) ((Map) dataSourceInfo.get("dataSourceType")).get("name");
        Integer currentDataSourceId = (Integer) dataSourceInfo.get("id");
        dataSourceRequest.setLinkisDataSourceId(currentDataSourceId.longValue());
        cols = metaDataClient.getColumnsByDataSource(clusterName, userName, currentDataSourceId.longValue(), database, table).getContent();
        dataSourceRequest.setLinkisDataSourceName(dataSourceInfoName);
        dataSourceRequest.setLinkisDataSourceType(dataSourceInfoType);
    }
    // For one or more fields
    if (StringUtils.isBlank(col)) {
        LOGGER.info("Table count check.");
    } else {
        boolean blackList = false;
        List<String> colsInDatasource = new ArrayList<>();
        if (col.startsWith(SpecCharEnum.MINUS.getValue())) {
            blackList = true;
            col = col.replace(SpecCharEnum.MINUS.getValue(), "");
        }
        if (col.contains(SpecCharEnum.LEFT_BRACKET.getValue()) && col.contains(SpecCharEnum.RIGHT_BRACKET.getValue())) {
            col = col.replace(SpecCharEnum.LEFT_BRACKET.getValue(), "").replace(SpecCharEnum.RIGHT_BRACKET.getValue(), "");
            for (String currentCol : col.split(SpecCharEnum.COMMA.getValue())) {
                colsInDatasource.add(currentCol);
            }
        } else {
            colsInDatasource.add(col);
        }
        if (blackList) {
            for (ColumnInfoDetail columnInfoDetail : cols) {
                if (colsInDatasource.contains(columnInfoDetail.getFieldName())) {
                    continue;
                } else {
                    DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest(columnInfoDetail.getFieldName(), columnInfoDetail.getDataType());
                    dataSourceColumnRequests.add(dataSourceColumnRequest);
                }
            }
        } else {
            for (String colName : colsInDatasource) {
                String type = cols.stream().filter(field -> field.getFieldName().equals(colName)).map(ColumnInfoDetail::getDataType).collect(Collectors.joining());
                DataSourceColumnRequest dataSourceColumnRequest = new DataSourceColumnRequest(colName, type);
                dataSourceColumnRequests.add(dataSourceColumnRequest);
            }
        }
        dataSourceRequest.setBlackList(blackList);
    }
    dataSourceRequest.setClusterName(clusterName);
    dataSourceRequest.setDbName(database);
    dataSourceRequest.setTableName(table);
    dataSourceRequest.setColNames(dataSourceColumnRequests);
    dataSourceRequest.setFilter(filter);
    dataSourceRequest.setProxyUser(userName);
    dataSourceRequests.add(dataSourceRequest);
    this.addRuleRequest.setDatasource(dataSourceRequests);
}

Example 19 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class ProjectBatchServiceImpl method modifyRuleMetric.

private void modifyRuleMetric(ExcelRuleMetric excelRuleMetric, RuleMetric ruleMetric, String userName) throws UnExpectedRequestException, PermissionDeniedRequestException {
    // Check en code existence.
    RuleMetric ruleMetricInDb = ruleMetricDao.findByEnCode(excelRuleMetric.getEnCode());
    if (ruleMetricInDb != null && ruleMetricInDb.getId().longValue() != ruleMetric.getId().longValue()) {
        throw new UnExpectedRequestException("Rule Metric [EN_CODE=" + excelRuleMetric.getEnCode() + "] {&DOES_NOT_EXIST}");
    }
    LOGGER.info("Start to modify rule metric, modify request: [{}], user: [{}]", excelRuleMetric.toString(), userName);
    User loginUser = userDao.findByUsername(userName);
    List<UserRole> userRoles = userRoleDao.findByUser(loginUser);
    Integer roleType = roleService.getRoleType(userRoles);
    if (roleType.equals(RoleDefaultTypeEnum.ADMIN.getCode())) {
        LOGGER.info("First level(created by SYS_ADMIN) indicator will be modified soon.");
    } else if (roleType.equals(RoleDefaultTypeEnum.DEPARTMENT_ADMIN.getCode())) {
        LOGGER.info("Second level(created by DEPARTMENT_ADMIN) indicator will be modified soon.");
        if (ruleMetric.getLevel().equals(RuleMetricLevelEnum.DEFAULT_METRIC.getCode())) {
            throw new PermissionDeniedRequestException("User {&HAS_NO_PERMISSION_TO_ACCESS}", 403);
        }
        List<Department> managedDepartment = new ArrayList<>();
        for (UserRole userRole : userRoles) {
            Department department = userRole.getRole().getDepartment();
            if (department != null) {
                managedDepartment.add(department);
            }
        }
        RuleMetricDepartmentUser ruleMetricDepartmentUser = ruleMetricDepartmentUserDao.findByRuleMetric(ruleMetric);
        if (ruleMetricDepartmentUser != null && managedDepartment.contains(ruleMetricDepartmentUser.getDepartment())) {
            LOGGER.info("Rule metric[{}] comes from department: {}", ruleMetric.toString(), ruleMetricDepartmentUser.getDepartment().getName());
        } else {
            throw new PermissionDeniedRequestException("User {&HAS_NO_PERMISSION_TO_ACCESS}", 403);
        }
    } else {
        LOGGER.info("Third level(created by PROJECTOR) indicator will be modified soon.");
        if (!ruleMetric.getLevel().equals(RuleMetricLevelEnum.PERSONAL_METRIC.getCode()) || !ruleMetric.getCreateUser().equals(loginUser.getUserName())) {
            throw new PermissionDeniedRequestException("User {&HAS_NO_PERMISSION_TO_ACCESS}", 403);
        }
    }
    ruleMetric.setName(excelRuleMetric.getName());
    ruleMetric.setMetricDesc(excelRuleMetric.getMetricDesc());
    ruleMetric.setSubSystemName(excelRuleMetric.getSubSystemName());
    ruleMetric.setFullCnName(excelRuleMetric.getFullCnName());
    ruleMetric.setModifyUser(userName);
    ruleMetric.setModifyTime(ExecutionManagerImpl.PRINT_TIME_FORMAT.format(new Date()));
    ruleMetric.setType(excelRuleMetric.getType());
    ruleMetric.setFrequency(Integer.parseInt(excelRuleMetric.getFrequency()));
    ruleMetric.setEnCode(excelRuleMetric.getEnCode());
    ruleMetric.setDepartmentName(excelRuleMetric.getDepartmentName());
    ruleMetric.setDevDepartmentName(excelRuleMetric.getDevDepartmentName());
    ruleMetric.setOpsDepartmentName(excelRuleMetric.getOpsDepartmentName());
    ruleMetric.setAvailable(excelRuleMetric.getAvailable());
    ruleMetricDao.add(ruleMetric);
}

Example 20 with UnExpectedRequestException

use of com.webank.wedatasphere.qualitis.exception.UnExpectedRequestException in project Qualitis by WeBankFinTech.

the class ExecutionManagerImpl method submitApplication.

/**
 * Submit job to linkis
 */
@Override
public List<TaskSubmitResult> submitApplication(List<Rule> rules, String nodeName, String createTime, String user, String database, StringBuffer partition, Date date, Application application, String cluster, String startupParam, String setFlag, Map<String, String> execParams, StringBuffer runDate, Map<Long, Map> dataSourceMysqlConnect) throws ArgumentException, TaskTypeException, ConvertException, DataQualityTaskException, RuleVariableNotSupportException, RuleVariableNotFoundException, JobSubmitException, ClusterInfoNotConfigException, IOException, UnExpectedRequestException, MetaDataAcquireFailedException {
    String csId = rules.iterator().next().getCsId();
    // Check if cluster supported
    LOGGER.info("Start to collect rule to clusters");
    Map<String, List<Rule>> clusterNameMap = getRuleCluster(rules);
    LOGGER.info("Succeed to classify rules by cluster, cluster map: {}", clusterNameMap);
    if (StringUtils.isNotBlank(cluster)) {
        LOGGER.info("When pick up a cluster, these datasources of rules must be from one cluster. Now start to put into the specify cluster.\n");
        putAllRulesIntoSpecifyCluster(clusterNameMap, cluster);
        LOGGER.info("Success to put into the specify cluster.\n");
    }
    List<TaskSubmitResult> taskSubmitResults = new ArrayList<>();
    for (String clusterName : clusterNameMap.keySet()) {
        List<Rule> clusterRules = clusterNameMap.get(clusterName);
        if (StringUtils.isNotBlank(cluster)) {
            clusterName = cluster;
        }
        ClusterInfo clusterInfo = clusterInfoDao.findByClusterName(clusterName);
        LOGGER.info("Start to check cluster config.");
        if (clusterInfo == null) {
            throw new ClusterInfoNotConfigException(clusterName + " {&DOES_NOT_EXIST}");
        }
        LOGGER.info("Succeed to pass the check of cluster config. All cluster of rules are configured");
        // Divide rule into tasks
        List<DataQualityTask> tasks = TaskDividerFactory.getDivider().divide(clusterRules, application.getId(), createTime, partition.toString(), date, database, user, taskExecuteLimitConfig.getTaskExecuteRuleSize());
        LOGGER.info("Succeed to divide application into tasks. result: {}", tasks);
        // Save divided tasks
        saveDividedTask(tasks, clusterInfo, rules, application, createTime);
        // Convert tasks into job
        List<DataQualityJob> jobList = new ArrayList<>();
        for (DataQualityTask task : tasks) {
            DataQualityJob job = templateConverterFactory.getConverter(task).convert(task, date, setFlag, execParams, runDate.toString(), clusterInfo.getClusterType(), dataSourceMysqlConnect);
            job.setUser(task.getUser());
            jobList.add(job);
            List<Long> ruleIdList = task.getRuleTaskDetails().stream().map(r -> r.getRule().getId()).collect(Collectors.toList());
            LOGGER.info("Succeed to convert rule_id: {} into code. code: {}", ruleIdList, job.getJobCode());
        }
        LOGGER.info("Succeed to convert all template into codes. codes: {}", jobList);
        // Submit job to linkis
        List<JobSubmitResult> submitResults = new ArrayList<>();
        for (DataQualityJob job : jobList) {
            String code = String.join("\n", job.getJobCode());
            String proxy = job.getUser();
            Long taskId = job.getTaskId();
            // Compatible with new and old submission interfaces.
            JobSubmitResult result = null;
            boolean engineReUse = false;
            if (StringUtils.isNotBlank(startupParam)) {
                String[] startupParams = startupParam.split(SpecCharEnum.DIVIDER.getValue());
                for (String param : startupParams) {
                    if (StringUtils.isEmpty(param)) {
                        continue;
                    }
                    String[] paramStrs = param.split("=");
                    if (paramStrs.length < 2) {
                        continue;
                    }
                    String key = paramStrs[0];
                    String value = paramStrs[1];
                    if ("engine_reuse".equals(key)) {
                        if ("true".equals(value)) {
                            engineReUse = true;
                            startupParam = startupParam.replace("engine_reuse=true", "");
                        } else {
                            engineReUse = false;
                            startupParam = startupParam.replace("engine_reuse=false", "");
                        }
                        break;
                    }
                }
            }
            if (clusterInfo.getClusterType().endsWith(LINKIS_ONE_VERSION)) {
                result = abstractJobSubmitter.submitJobNew(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam(), engineReUse);
            } else {
                result = abstractJobSubmitter.submitJob(code, linkisConfig.getEngineName(), StringUtils.isNotBlank(proxy) ? proxy : user, clusterInfo.getLinkisAddress(), clusterName, taskId, csId, nodeName, StringUtils.isNotBlank(startupParam) ? startupParam : job.getStartupParam());
            }
            if (result != null) {
                submitResults.add(result);
            } else {
                Task taskInDb = taskDao.findById(taskId);
                taskInDb.setStatus(TaskStatusEnum.TASK_NOT_EXIST.getCode());
                taskDao.save(taskInDb);
                taskSubmitResults.add(new TaskSubmitResult(application.getId(), null, clusterInfo.getClusterName()));
            }
        }
        // Rewrite task remote ID.
        rewriteTaskRemoteInfo(submitResults, taskSubmitResults, application.getId(), clusterInfo.getClusterName());
    }
    return taskSubmitResults;
}