Examples with ValidationRule com.b2international.snowowl.core.validation.rule.ValidationRule used on opensource projects

Example 1 with ValidationRule

use of com.b2international.snowowl.core.validation.rule.ValidationRule in project snow-owl by b2ihealthcare.

the class ValidationPlugin method preRun.

@Override
public void preRun(SnowOwlConfiguration configuration, Environment env) throws Exception {
    if (env.isServer()) {
        final ObjectMapper mapper = env.service(ObjectMapper.class);
        final Index validationIndex = Indexes.createIndex(VALIDATIONS_INDEX, mapper, new Mappings(ValidationIssue.class, ValidationRule.class, ValidationWhiteList.class), env.service(IndexSettings.class).forIndex(env.service(RepositoryConfiguration.class).getIndexConfiguration(), VALIDATIONS_INDEX));
        final ValidationRepository repository = new ValidationRepository(validationIndex);
        env.services().registerService(ValidationRepository.class, repository);
        // register always available validation rule evaluators
        ValidationRuleEvaluator.Registry.register(new GroovyScriptValidationRuleEvaluator(env.getConfigPath()));
        // initialize validation thread pool
        final ValidationConfiguration validationConfig = configuration.getModuleConfig(ValidationConfiguration.class);
        int numberOfValidationThreads = validationConfig.getNumberOfValidationThreads();
        int maxConcurrentExpensiveJobs = validationConfig.getMaxConcurrentExpensiveJobs();
        int maxConcurrentNormalJobs = validationConfig.getMaxConcurrentNormalJobs();
        env.services().registerService(ValidationConfiguration.class, validationConfig);
        env.services().registerService(ValidationThreadPool.class, new ValidationThreadPool(numberOfValidationThreads, maxConcurrentExpensiveJobs, maxConcurrentNormalJobs));
        env.services().registerService(ValidationIssueDetailExtensionProvider.class, new ValidationIssueDetailExtensionProvider(env.service(ClassPathScanner.class)));
        final List<File> listOfFiles = Arrays.asList(env.getConfigPath().toFile().listFiles());
        final Set<File> validationRuleFiles = Sets.newHashSet();
        final Pattern validationFilenamePattern = Pattern.compile("(validation-rules)-(\\w+).(json)");
        for (File file : listOfFiles) {
            final String fileName = file.getName();
            final Matcher match = validationFilenamePattern.matcher(fileName);
            if (match.matches()) {
                validationRuleFiles.add(file);
            }
        }
        final List<ValidationRule> availableRules = Lists.newArrayList();
        for (File validationRulesFile : validationRuleFiles) {
            LOG.info("Synchronizing validation rules from file: " + validationRulesFile);
            availableRules.addAll(mapper.readValue(validationRulesFile, new TypeReference<List<ValidationRule>>() {
            }));
        }
        repository.write(writer -> {
            final Map<String, ValidationRule> existingRules = Maps.uniqueIndex(ValidationRequests.rules().prepareSearch().all().buildAsync().getRequest().execute(env), ValidationRule::getId);
            // index all rules from the file, this will update existing rules as well
            final Set<String> ruleIds = newHashSet();
            for (ValidationRule rule : availableRules) {
                writer.put(rule);
                ruleIds.add(rule.getId());
            }
            // delete rules and associated issues
            Set<String> rulesToDelete = Sets.difference(existingRules.keySet(), ruleIds);
            if (!rulesToDelete.isEmpty()) {
                final Set<String> issuesToDelete = newHashSet(writer.searcher().search(Query.select(String.class).from(ValidationIssue.class).fields(ValidationIssue.Fields.ID).where(Expressions.builder().filter(Expressions.matchAny(ValidationIssue.Fields.RULE_ID, rulesToDelete)).build()).limit(Integer.MAX_VALUE).build()).getHits());
                writer.removeAll(ImmutableMap.<Class<?>, Set<String>>of(ValidationRule.class, rulesToDelete, ValidationIssue.class, issuesToDelete));
            }
            writer.commit();
            return null;
        });
    }
}

Example 2 with ValidationRule

use of com.b2international.snowowl.core.validation.rule.ValidationRule in project snow-owl by b2ihealthcare.

the class ValidateRequest method doValidate.

private ValidationResult doValidate(BranchContext context, Writer index) throws IOException {
    final String branchPath = context.path();
    ValidationRuleSearchRequestBuilder req = ValidationRequests.rules().prepareSearch();
    TerminologyResource resource = context.service(TerminologyResource.class);
    ResourceURI resourceURI = resource.getResourceURI(branchPath);
    if (!CompareUtils.isEmpty(ruleIds)) {
        req.filterByIds(ruleIds);
    }
    final ValidationRules rules = req.all().build().execute(context);
    final ValidationThreadPool pool = context.service(ValidationThreadPool.class);
    final BlockingQueue<IssuesToPersist> issuesToPersistQueue = Queues.newLinkedBlockingDeque();
    final List<Promise<Object>> validationPromises = Lists.newArrayList();
    // evaluate selected rules
    for (ValidationRule rule : rules) {
        checkArgument(rule.getCheckType() != null, "CheckType is missing for rule " + rule.getId());
        final ValidationRuleEvaluator evaluator = ValidationRuleEvaluator.Registry.get(rule.getType());
        if (evaluator != null) {
            validationPromises.add(pool.submit(rule.getCheckType(), () -> {
                Stopwatch w = Stopwatch.createStarted();
                try {
                    LOG.info("Executing rule '{}'...", rule.getId());
                    final List<?> evaluationResponse = evaluator.eval(context, rule, ruleParameters);
                    issuesToPersistQueue.offer(new IssuesToPersist(rule.getId(), evaluationResponse));
                    LOG.info("Execution of rule '{}' successfully completed in '{}'.", rule.getId(), w);
                // TODO report successfully executed validation rule
                } catch (Exception e) {
                    // TODO report failed validation rule
                    LOG.error("Execution of rule '{}' failed after '{}'.", rule.getId(), w, e);
                }
            }));
        }
    }
    final Set<String> ruleIds = rules.stream().map(ValidationRule::getId).collect(Collectors.toSet());
    final Multimap<String, ComponentIdentifier> whiteListedEntries = fetchWhiteListEntries(context, ruleIds);
    final Promise<List<Object>> promise = Promise.all(validationPromises);
    while (!promise.isDone() || !issuesToPersistQueue.isEmpty()) {
        if (!issuesToPersistQueue.isEmpty()) {
            final Collection<IssuesToPersist> issuesToPersist = newArrayList();
            issuesToPersistQueue.drainTo(issuesToPersist);
            if (!issuesToPersist.isEmpty()) {
                final List<String> rulesToPersist = issuesToPersist.stream().map(itp -> itp.ruleId).collect(Collectors.toList());
                LOG.info("Persisting issues generated by rules '{}'...", rulesToPersist);
                // persist new issues generated by rules so far, extending them using the Issue Extension API
                int persistedIssues = 0;
                final Multimap<String, ValidationIssue> issuesToExtendWithDetailsByToolingId = HashMultimap.create();
                for (IssuesToPersist ruleIssues : Iterables.consumingIterable(issuesToPersist)) {
                    final String ruleId = ruleIssues.ruleId;
                    final List<ValidationIssue> existingRuleIssues = ValidationRequests.issues().prepareSearch().all().filterByResourceUri(resourceURI).filterByRule(ruleId).build().execute(context).getItems();
                    final Set<String> issueIdsToDelete = Sets.newHashSet();
                    final Map<ComponentIdentifier, ValidationIssue> existingIsssuesByComponentIdentifier = new HashMap<>();
                    for (ValidationIssue issue : existingRuleIssues) {
                        if (existingIsssuesByComponentIdentifier.containsKey(issue.getAffectedComponent())) {
                            issueIdsToDelete.add(issue.getId());
                        } else {
                            existingIsssuesByComponentIdentifier.put(issue.getAffectedComponent(), issue);
                        }
                    }
                    // remove all processed whitelist entries
                    final Collection<ComponentIdentifier> ruleWhiteListEntries = whiteListedEntries.removeAll(ruleId);
                    final String toolingId = rules.stream().filter(rule -> ruleId.equals(rule.getId())).findFirst().get().getToolingId();
                    for (ValidationIssueDetails issueDetails : ruleIssues.issueDetails) {
                        final ValidationIssue validationIssue;
                        ComponentIdentifier componentIdentifier = issueDetails.affectedComponentId;
                        if (!existingIsssuesByComponentIdentifier.containsKey(componentIdentifier)) {
                            validationIssue = new ValidationIssue(UUID.randomUUID().toString(), ruleId, ComponentURI.of(resourceURI, componentIdentifier), ruleWhiteListEntries.contains(componentIdentifier));
                        } else {
                            final ValidationIssue issueToCopy = existingIsssuesByComponentIdentifier.get(componentIdentifier);
                            validationIssue = new ValidationIssue(issueToCopy.getId(), issueToCopy.getRuleId(), ComponentURI.of(resourceURI, issueToCopy.getAffectedComponent()), ruleWhiteListEntries.contains(issueToCopy.getAffectedComponent()));
                            existingIsssuesByComponentIdentifier.remove(componentIdentifier);
                        }
                        validationIssue.setDetails(ValidationIssueDetails.HIGHLIGHT_DETAILS, issueDetails.stylingDetails);
                        issuesToExtendWithDetailsByToolingId.put(toolingId, validationIssue);
                        persistedIssues++;
                    }
                    existingRuleIssues.stream().filter(issue -> existingIsssuesByComponentIdentifier.containsKey(issue.getAffectedComponent())).forEach(issue -> issueIdsToDelete.add(issue.getId()));
                    if (!issueIdsToDelete.isEmpty()) {
                        index.removeAll(Collections.singletonMap(ValidationIssue.class, issueIdsToDelete));
                    }
                }
                for (String toolingId : issuesToExtendWithDetailsByToolingId.keySet()) {
                    final ValidationIssueDetailExtension extensions = context.service(ValidationIssueDetailExtensionProvider.class).getExtensions(toolingId);
                    final Collection<ValidationIssue> issues = issuesToExtendWithDetailsByToolingId.removeAll(toolingId);
                    extensions.extendIssues(context, issues, ruleParameters);
                    for (ValidationIssue issue : issues) {
                        index.put(issue);
                    }
                }
                index.commit();
                LOG.info("Persisted '{}' issues generated by rules '{}'.", persistedIssues, rulesToPersist);
            }
        } else {
            try {
                // wait at least number of rules * 50ms for the next responses
                Thread.sleep(Math.min(((long) ruleIds.size()) * 100, POLL_INTERVAL_MAX));
            } catch (InterruptedException e) {
                throw new SnowowlRuntimeException(e);
            }
        }
    }
    // TODO return ValidationResult object with status and new issue IDs as set
    return new ValidationResult(context.info().id(), context.path());
}

Example 3 with ValidationRule

use of com.b2international.snowowl.core.validation.rule.ValidationRule in project snow-owl by b2ihealthcare.

the class BaseValidationTest method indexRule.

/* Looks up the rule in the validation-rules.json file and indexes it */
protected final void indexRule(String ruleId) throws Exception {
    Preconditions.checkArgument(!Strings.isNullOrEmpty(rulesJsonFile), "validation-rules.json file path must be specified to use this method");
    URL rulesJson = getClass().getClassLoader().getResource(rulesJsonFile);
    try (InputStream in = rulesJson.openStream()) {
        MappingIterator<ValidationRule> it = context.service(ObjectMapper.class).readerFor(ValidationRule.class).readValues(in);
        while (it.hasNext()) {
            final ValidationRule rule = it.next();
            if (ruleId.equals(rule.getId())) {
                indexDocument(rule);
                return;
            }
        }
    }
}

Example 4 with ValidationRule

use of com.b2international.snowowl.core.validation.rule.ValidationRule in project snow-owl by b2ihealthcare.

the class RepositoryValidationRestService method getValidationResults.

@Operation(summary = "Retrieve the validation issues from a completed validation on a branch. Output may differ by the chosen content type.")
@ApiResponses({ @ApiResponse(responseCode = "200", description = "OK"), @ApiResponse(responseCode = "404", description = "Branch not found") })
@RequestMapping(value = "/validations/{validationId}/issues", method = RequestMethod.GET, produces = { AbstractRestService.JSON_MEDIA_TYPE, AbstractRestService.CSV_MEDIA_TYPE })
@ResponseBody
public Promise<Collection<Object>> getValidationResults(@Parameter(description = "The unique validation identifier.") @PathVariable(value = "validationId") final String validationId, @Parameter(description = "The search key to use for retrieving the next page of results") @RequestParam(value = "searchAfter", required = false) final String searchAfter, @Parameter(description = "The maximum number of items to return") @RequestParam(value = "limit", defaultValue = "50", required = false) final int limit, @Parameter(hidden = true) @RequestHeader(value = HttpHeaders.ACCEPT, defaultValue = AbstractRestService.JSON_MEDIA_TYPE, required = false) final String contentType) {
    final IEventBus bus = getBus();
    return getValidationRun(validationId).thenWith(validationJob -> {
        final ResourceURI codeSystemURI = getCodeSystemURIFromJob(validationJob);
        if (AbstractRestService.CSV_MEDIA_TYPE.equals(contentType)) {
            return ValidationRequests.issues().prepareSearch().isWhitelisted(false).all().filterByResourceUri(codeSystemURI).sortBy(Sort.fieldAsc(ValidationIssue.Fields.RULE_ID)).buildAsync().execute(bus).then(issues -> {
                final Set<String> rulesToFetch = issues.stream().map(ValidationIssue::getRuleId).collect(Collectors.toSet());
                final Map<String, String> ruleDescriptionById = ValidationRequests.rules().prepareSearch().all().filterByIds(rulesToFetch).buildAsync().execute(bus).getSync(1, TimeUnit.MINUTES).stream().collect(Collectors.toMap(ValidationRule::getId, ValidationRule::getMessageTemplate));
                return issues.stream().map(issue -> {
                    final String ruleId = issue.getRuleId();
                    final String ruleDescription = ruleDescriptionById.get(ruleId);
                    final String affectedComponentLabel = Iterables.getFirst(issue.getAffectedComponentLabels(), "No label found");
                    final String affectedComponentId = issue.getAffectedComponent().getComponentId();
                    return new ValidationIssueReport(ruleId, ruleDescription, affectedComponentId, affectedComponentLabel);
                }).collect(Collectors.toList());
            });
        } else {
            return ValidationRequests.issues().prepareSearch().isWhitelisted(false).setLimit(limit).setSearchAfter(searchAfter).filterByResourceUri(codeSystemURI).buildAsync().execute(bus).then(issues -> issues.getItems().stream().collect(Collectors.toList()));
        }
    });
}

Example 5 with ValidationRule

use of com.b2international.snowowl.core.validation.rule.ValidationRule in project snow-owl by b2ihealthcare.

the class ValidationRuleApiTest method createRule.

@Test
public void createRule() throws Exception {
    final String ruleId = ValidationRequests.rules().prepareCreate().setId(UUID.randomUUID().toString()).setToolingId("TerminologyToolingId").setMessageTemplate("Error message").setSeverity(Severity.ERROR).setType("snomed-query").setImplementation("*").buildAsync().getRequest().execute(context);
    final ValidationRule rule = ValidationRequests.rules().prepareGet(ruleId).buildAsync().getRequest().execute(context);
    assertThat(rule.getToolingId()).isEqualTo("TerminologyToolingId");
    assertThat(rule.getMessageTemplate()).isEqualTo("Error message");
    assertThat(rule.getSeverity()).isEqualTo(Severity.ERROR);
    assertThat(rule.getType()).isEqualTo("snomed-query");
    assertThat(rule.getImplementation()).isEqualTo("*");
}