Examples with RuleDataSource com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource used on opensource projects

Example 11 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method constructFileRequest.

private AddFileRuleRequest constructFileRequest(Rule rule, RuleGroup ruleGroup) {
    AddFileRuleRequest addFileRuleRequest = new AddFileRuleRequest();
    String newVersion = ruleGroup.getVersion();
    if (StringUtils.isEmpty(newVersion)) {
        String newRuleName = rule.getName() + "_copy_" + ruleGroup.getId();
        LOGGER.info("File rule start to be copied. Copied rule name: " + newRuleName);
        addFileRuleRequest.setRuleName(newRuleName);
        if (StringUtils.isNotBlank(rule.getCnName())) {
            addFileRuleRequest.setRuleCnName(rule.getCnName() + "_副本");
        }
    } else {
        String oldVersion = rule.getRuleGroup().getVersion();
        LOGGER.info("File rule start to be copied. Copied rule name: " + rule.getName() + "_" + ruleGroup.getVersion());
        if (StringUtils.isNotBlank(oldVersion) && rule.getName().endsWith(oldVersion)) {
            addFileRuleRequest.setRuleName(rule.getName().replace(oldVersion, newVersion));
            if (StringUtils.isNotBlank(rule.getCnName())) {
                addFileRuleRequest.setRuleCnName(rule.getCnName().replace(oldVersion, newVersion));
            }
        } else {
            addFileRuleRequest.setRuleName(rule.getName() + "_" + newVersion);
            if (StringUtils.isNotBlank(rule.getCnName())) {
                addFileRuleRequest.setRuleCnName(rule.getCnName() + "_" + newVersion);
            }
        }
    }
    String ruleDetail = rule.getDetail();
    addFileRuleRequest.setRuleDetail(ruleDetail);
    addFileRuleRequest.setAbortOnFailure(rule.getAbortOnFailure());
    addFileRuleRequest.setCsId(rule.getCsId());
    addFileRuleRequest.setRuleGroupId(ruleGroup.getId());
    addFileRuleRequest.setProjectId(ruleGroup.getProjectId());
    addFileRuleRequest.setAbortOnFailure(rule.getAbortOnFailure());
    for (RuleDataSource ruleDataSource : rule.getRuleDataSources()) {
        DataSourceRequest dataSourceRequest = new DataSourceRequest();
        String clusterName = ruleDataSource.getClusterName();
        String databaseName = ruleDataSource.getDbName();
        String tableName = ruleDataSource.getTableName();
        String filter = ruleDataSource.getFilter();
        dataSourceRequest.setClusterName(clusterName);
        dataSourceRequest.setDbName(databaseName);
        dataSourceRequest.setTableName(tableName);
        dataSourceRequest.setFilter(filter);
        addFileRuleRequest.setDatasource(dataSourceRequest);
        break;
    }
    List<FileAlarmConfigRequest> alarmVariable = new ArrayList<>();
    for (AlarmConfig alarmConfig : rule.getAlarmConfigs()) {
        FileAlarmConfigRequest fileAlarmConfigRequest = new FileAlarmConfigRequest();
        Double threshold = alarmConfig.getThreshold();
        Integer unit = alarmConfig.getFileOutputUnit();
        Integer alarmCompareType = alarmConfig.getCompareType();
        Integer alarmOutputName = alarmConfig.getFileOutputName();
        Integer checkTemplateName = alarmConfig.getCheckTemplate();
        fileAlarmConfigRequest.setFileOutputUnit(unit);
        fileAlarmConfigRequest.setCompareType(alarmCompareType);
        fileAlarmConfigRequest.setFileOutputName(alarmOutputName);
        fileAlarmConfigRequest.setCheckTemplate(checkTemplateName);
        fileAlarmConfigRequest.setThreshold(threshold);
        RuleMetric ruleMetric = alarmConfig.getRuleMetric();
        // Recod rule metric info (unique code).
        if (ruleMetric != null) {
            String enCode = ruleMetric.getEnCode();
            fileAlarmConfigRequest.setRuleMetricEnCode(enCode);
            fileAlarmConfigRequest.setUploadAbnormalValue(alarmConfig.getUploadAbnormalValue());
            fileAlarmConfigRequest.setUploadRuleMetricValue(alarmConfig.getUploadRuleMetricValue());
        }
        fileAlarmConfigRequest.setDeleteFailCheckResult(alarmConfig.getDeleteFailCheckResult());
        alarmVariable.add(fileAlarmConfigRequest);
    }
    addFileRuleRequest.setAlarm(true);
    addFileRuleRequest.setAlarmVariable(alarmVariable);
    return addFileRuleRequest;
}

Example 12 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class RuleNodeServiceImpl method updateImportedFileRule.

private void updateImportedFileRule(Rule rule, Rule ruleInDb, RuleGroup ruleGroup, Set<AlarmConfig> alarmConfigs, Set<RuleDataSource> ruleDataSources) {
    ruleInDb.setRuleGroup(ruleGroup);
    ruleInDb.setAlarm(rule.getAlarm());
    ruleInDb.setAbortOnFailure(rule.getAbortOnFailure());
    alarmConfigService.deleteByRule(ruleInDb);
    LOGGER.info("Succeed to delete all alarm_config. rule_id: {}", ruleInDb.getId());
    ruleDataSourceService.deleteByRule(ruleInDb);
    LOGGER.info("Succeed to delete all rule_dataSources. rule_id: {}", ruleInDb.getId());
    Rule updateRule = ruleDao.saveRule(ruleInDb);
    // Save file alarmconfig
    List<AlarmConfig> alarmConfigList = new ArrayList<>();
    for (AlarmConfig alarmConfig : alarmConfigs) {
        alarmConfig.setRule(updateRule);
        alarmConfigList.add(alarmConfig);
    }
    // Save file datasource
    List<RuleDataSource> ruleDataSourceList = new ArrayList<>();
    for (RuleDataSource ruleDataSource : ruleDataSources) {
        ruleDataSource.setProjectId(updateRule.getId());
        ruleDataSource.setRule(updateRule);
        ruleDataSourceList.add(ruleDataSource);
    }
    updateRule.setAlarmConfigs(new HashSet<>(alarmConfigDao.saveAllAlarmConfig(alarmConfigList)));
    updateRule.setRuleDataSources(new HashSet<>(ruleDataSourceDao.saveAllRuleDataSource(ruleDataSourceList)));
}

Example 13 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class OuterExecutionServiceImpl method customReSaveDateSource.

private void customReSaveDateSource(Rule currentRule, Map<String, String> execParams, String clusterName, Date date) throws UnExpectedRequestException, SemanticException, ParseException {
    String midTableAction = currentRule.getTemplate().getMidTableAction();
    if (StringUtils.isNotBlank(currentRule.getWhereContent())) {
        midTableAction = midTableAction.replace("${filter}", currentRule.getWhereContent());
    }
    // Replace placeholder with key and value for sql parse check.
    for (String key : execParams.keySet()) {
        midTableAction = midTableAction.replace("${" + key + "}", execParams.get(key));
    }
    midTableAction = DateExprReplaceUtil.replaceRunDate(date, midTableAction);
    // Save datasource in first execution(Only possible data sources related to fps).
    boolean firstExecution = CollectionUtils.isEmpty(currentRule.getRuleDataSources().stream().filter(ruleDataSource -> !ORIGINAL_INDEX.equals(ruleDataSource.getDatasourceIndex())).collect(Collectors.toSet()));
    if (!firstExecution) {
        return;
    }
    RuleDataSource originalRuleDataSource = currentRule.getRuleDataSources().stream().filter(ruleDataSource -> ORIGINAL_INDEX.equals(ruleDataSource.getDatasourceIndex())).iterator().next();
    if (StringUtils.isEmpty(clusterName)) {
        clusterName = originalRuleDataSource.getClusterName();
    }
    Map<String, List<String>> dbAndTables = new HashMap<>(2);
    // 0.16.0 for mysql oracle, hive parse, use druid.
    LOGGER.info("Parse sql: " + midTableAction);
    List<RuleMetric> ruleMetrics = currentRule.getAlarmConfigs().stream().map(AlarmConfig::getRuleMetric).distinct().collect(Collectors.toList());
    Set<String> ruleMetricNames = ruleMetrics.stream().map(RuleMetric::getName).collect(Collectors.toSet());
    for (String ruleMetricName : ruleMetricNames) {
        String cleanRuleMetricName = ruleMetricName.replace("-", "_");
        midTableAction = midTableAction.replace(ruleMetricName, cleanRuleMetricName);
    }
    LOGGER.info("Parse sql after replace rule metric names: " + midTableAction);
    if (originalRuleDataSource.getLinkisDataSourceId() != null) {
        DbType dbType = JdbcConstants.MYSQL;
        MySqlSchemaStatVisitor visitor = new MySqlSchemaStatVisitor();
        List<SQLStatement> stmtList = SQLUtils.parseStatements(midTableAction, dbType);
        for (int i = 0; i < stmtList.size(); i++) {
            SQLStatement stmt = stmtList.get(i);
            stmt.accept(visitor);
            Map<Name, TableStat> tableStatMap = visitor.getTables();
            dbAndTables = toDbAndTables(dbAndTables, tableStatMap);
        }
    } else {
        HiveSqlParser hiveSqlParser = new HiveSqlParser();
        dbAndTables = hiveSqlParser.checkSelectSqlAndGetDbAndTable(midTableAction);
    }
    LOGGER.info("Db and tables: " + dbAndTables.toString());
    for (String db : dbAndTables.keySet()) {
        List<RuleDataSource> ruleDataSources = new ArrayList<>();
        for (String table : dbAndTables.get(db)) {
            RuleDataSource ruleDataSource = new RuleDataSource();
            ruleDataSource.setClusterName(clusterName);
            ruleDataSource.setProjectId(currentRule.getProject().getId());
            ruleDataSource.setDbName(db.toLowerCase());
            ruleDataSource.setRule(currentRule);
            ruleDataSource.setTableName(table);
            ruleDataSource.setProxyUser(originalRuleDataSource.getProxyUser());
            ruleDataSource.setLinkisDataSourceId(originalRuleDataSource.getLinkisDataSourceId());
            ruleDataSource.setLinkisDataSourceName(originalRuleDataSource.getLinkisDataSourceName());
            ruleDataSource.setLinkisDataSourceVersionId(originalRuleDataSource.getLinkisDataSourceVersionId());
            ruleDataSources.add(ruleDataSource);
        }
        Set<RuleDataSource> ruleDataSourcesExist = currentRule.getRuleDataSources();
        ruleDataSourcesExist.addAll(ruleDataSourceDao.saveAllRuleDataSource(ruleDataSources).stream().collect(Collectors.toSet()));
        currentRule.setRuleDataSources(ruleDataSourcesExist);
        ruleDao.saveRule(currentRule);
    }
}

Example 14 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class MetaDataServiceImpl method addMultiDbRules.

@Override
@Transactional(rollbackFor = { Exception.class, UnExpectedRequestException.class })
public String addMultiDbRules(MulDbRequest request) throws UnExpectedRequestException, MetaDataAcquireFailedException {
    MulDbRequest.checkRequst(request);
    String loginUser = HttpUtils.getUserName(httpServletRequest);
    LOGGER.info("Start to get all tables with source database and target database.");
    List<String> sourceTableName = new ArrayList<>();
    List<String> targetTableName = new ArrayList<>();
    if (request.getSourceLinkisDataSourceId() == null) {
        GetUserTableByDbIdRequest sourceRequest = new GetUserTableByDbIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getSourceDb());
        sourceRequest.setProxyUser(request.getProxyUser());
        GeneralResponse<GetAllResponse<TableInfoDetail>> sourceTableInfos = getUserTableByDbId(sourceRequest);
        if (sourceTableInfos.getData().getTotal() <= 0) {
            throw new UnExpectedRequestException("Source database has no tables.");
        }
        sourceTableName.addAll(sourceTableInfos.getData().getData().stream().map(TableInfoDetail::getTableName).collect(Collectors.toList()));
    } else {
        Map response = getTablesByDataSource(request.getClusterName(), request.getProxyUser(), request.getSourceLinkisDataSourceId(), request.getSourceDb());
        List<String> tables = (List<String>) response.get("tables");
        for (String table : tables) {
            sourceTableName.add(table);
        }
    }
    if (request.getTargetLinkisDataSourceId() == null) {
        GetUserTableByDbIdRequest targetRequest = new GetUserTableByDbIdRequest(0, Integer.MAX_VALUE, request.getClusterName(), request.getTargetDb());
        targetRequest.setProxyUser(request.getProxyUser());
        GeneralResponse<GetAllResponse<TableInfoDetail>> targetTableInfos = getUserTableByDbId(targetRequest);
        if (targetTableInfos.getData().getTotal() <= 0) {
            throw new UnExpectedRequestException("Target database has no tables.");
        }
        targetTableName.addAll(targetTableInfos.getData().getData().stream().map(TableInfoDetail::getTableName).collect(Collectors.toList()));
    } else {
        Map response = getTablesByDataSource(request.getClusterName(), request.getProxyUser(), request.getTargetLinkisDataSourceId(), request.getTargetDb());
        List<String> tables = (List<String>) response.get("tables");
        for (String table : tables) {
            targetTableName.add(table);
        }
    }
    // Collect the table with the same name that exists in the source database and the target database.
    sourceTableName.retainAll(targetTableName);
    Set<String> ruleFailedTable = new HashSet<>();
    Long projectId = request.getProjectId();
    List<String> ruleTables = new ArrayList<>(ruleDataSourceDao.findByProjectId(projectId).stream().map(RuleDataSource::getTableName).distinct().collect(Collectors.toList()));
    // Filter tables for which rules have been created.
    sourceTableName = sourceTableName.stream().filter(tableName -> !ruleTables.contains(tableName)).collect(Collectors.toList());
    // Rule name index starts with the last created rule.
    int ruleIndex = ruleDao.findByProject(projectDao.findById(projectId)).size();
    // Filter tables with black list.
    sourceTableName = filterTablesWithBlackList(request.getBlackList(), sourceTableName);
    // White list.
    List<String> whiteList = request.getWhiteList();
    // Filter special table to compare with accuracy template.
    List<FilterRequest> filterRequests = request.getFilterRequests();
    List<FilterRequest> sameTableFilterRequests = filterRequests.stream().filter(filterRequest -> filterRequest.getSourceTable().equals(filterRequest.getTargetTable())).collect(Collectors.toList());
    List<String> filterSameTableName = sameTableFilterRequests.stream().map(FilterRequest::getSourceTable).collect(Collectors.toList());
    sourceTableName.removeAll(filterSameTableName);
    filterRequests.removeAll(sameTableFilterRequests);
    List<String> filterDiffTableName = filterRequests.stream().map(filterRequest -> filterRequest.getSourceTable() + SpecCharEnum.COLON.getValue() + filterRequest.getTargetTable()).collect(Collectors.toList());
    whiteList.removeAll(filterDiffTableName);
    filterRequests.addAll(sameTableFilterRequests);
    // Check flag: create rules in a loop, and only do permission verification for the first time.
    boolean check = true;
    int sourceTableSize = sourceTableName.size();
    int filterRequestsSize = filterRequests.size();
    int total = ruleIndex + sourceTableSize + whiteList.size() + filterRequestsSize;
    int i = ruleIndex;
    for (; i < ruleIndex + sourceTableSize; i++) {
        String currentSameTable = sourceTableName.get(i - ruleIndex);
        try {
            AddMultiSourceRuleRequest addMultiSourceRuleRequest = constructRequest(request, null, currentSameTable, currentSameTable, i, total, loginUser);
            LOGGER.info("Start to add {}th multi source rule.", i);
            if (i > 0) {
                check = false;
            }
            multiSourceRuleService.addMultiSourceRule(addMultiSourceRuleRequest, check);
        } catch (Exception e) {
            LOGGER.error("One of rule failed to add, rule index:[{}], table name: [{}], exception:{}", i, currentSameTable, e.getMessage());
            ruleFailedTable.add(currentSameTable);
        }
        LOGGER.info("Finish to add {}th multi source rule.", i);
    }
    // With white list.
    withWhiteListAndFilterRequest(i, ruleIndex, sourceTableSize, total, whiteList, filterRequests, request, loginUser, check);
    // Record the table that failed to create a rule.
    saveRuleFailedTableInLabel(ruleFailedTable, projectId);
    // projectEventService.record(projectId, loginUser, "create multi-source rule with dbs", "Success: " + (total - ruleFailedTable.size()) + "; Failed: " + ruleFailedTable.size(), EventTypeEnum.MODIFY_PROJECT.getCode());
    return request.getSourceDb() + " vs " + request.getSourceDb() + ": " + "{&CONTINUE_CREATE_RULES}";
}

Example 15 with RuleDataSource

use of com.webank.wedatasphere.qualitis.rule.entity.RuleDataSource in project Qualitis by WeBankFinTech.

the class PredicateTest method test.

@Test
@Transactional
public void test() {
    String user = "v_wblwyan_test1";
    /*
     * ProjectUserDao测试
     */
    DataSourceQo param = new DataSourceQo();
    param.setUser(user);
    param.setUserType(new Integer[] { ProjectUserPermissionEnum.CREATOR.getCode() });
    List<ProjectUser> projectUsers = dao.findByUsernameAndPermissionsIn(param);
    assertTrue(projectUsers.isEmpty());
    param.setUserType(new Integer[] { ProjectUserPermissionEnum.CREATOR.getCode() });
    projectUsers = dao.findByUsernameAndPermissionsIn(param);
    assertTrue(projectUsers.isEmpty());
    param.setUserType(null);
    projectUsers = dao.findByUsernameAndPermissionsIn(param);
    assertTrue(projectUsers.isEmpty());
    param.setUserType(new Integer[] {});
    projectUsers = dao.findByUsernameAndPermissionsIn(param);
    assertTrue(projectUsers.isEmpty());
    /*
     *TaskDataSourceDao 测试
     */
    List<TaskDataSource> taskDataSources = taskDataSourceDao.findByUserAndDataSource(user, null, null, null, 0, 5);
    assertTrue(taskDataSources.isEmpty());
    taskDataSources = taskDataSourceDao.findByUserAndDataSource(user, "clusterName_test", null, null, 0, 5);
    assertTrue(taskDataSources.isEmpty());
    taskDataSources = taskDataSourceDao.findByUserAndDataSource(user, "clusterName_test", "databaseName_test", null, 0, 5);
    assertTrue(taskDataSources.isEmpty());
    taskDataSources = taskDataSourceDao.findByUserAndDataSource(user, "clusterName_test", "databaseName_test", "tableName_test", 0, 5);
    assertTrue(taskDataSources.isEmpty());
    long count = taskDataSourceDao.countByUserAndDataSource(user, "clusterName_test", "databaseName_test", null);
    assertEquals(0, count);
    /*
     *RuleDataSourceDao 测试
     */
    List<RuleDataSource> ruleDataSources = ruleDataSourceDao.findByProjectUser((long) Integer.MAX_VALUE, null, null, null);
    assertTrue(ruleDataSources.isEmpty());
    ruleDataSources = ruleDataSourceDao.findByProjectUser((long) Integer.MAX_VALUE, "clusterName_test", null, null);
    assertTrue(ruleDataSources.isEmpty());
    ruleDataSources = ruleDataSourceDao.findByProjectUser((long) Integer.MAX_VALUE, "clusterName_test", "databaseName_test", null);
    assertTrue(ruleDataSources.isEmpty());
    ruleDataSources = ruleDataSourceDao.findByProjectUser((long) Integer.MAX_VALUE, "clusterName_test", "databaseName_test", "tableName_test");
    assertTrue(ruleDataSources.isEmpty());
}