Examples with ValidationIssue com.b2international.snowowl.core.validation.issue.ValidationIssue used on opensource projects

Example 1 with ValidationIssue

use of com.b2international.snowowl.core.validation.issue.ValidationIssue in project snow-owl by b2ihealthcare.

the class SnomedValidationIssueDetailExtension method extendIssueDetails.

private void extendIssueDetails(BranchContext context, Collection<ValidationIssue> issues) {
    final RevisionSearcher searcher = context.service(RevisionSearcher.class);
    final Multimap<String, ValidationIssue> issuesByComponentId = Multimaps.index(issues, issue -> issue.getAffectedComponent().getComponentId());
    final Multimap<ComponentCategory, String> issueComponentIdsByComponentCategory = HashMultimap.create();
    issues.stream().forEach(issue -> {
        final ComponentCategory componentCategory = getComponentCategory(issue.getAffectedComponent().getComponentType());
        issueComponentIdsByComponentCategory.put(componentCategory, issue.getAffectedComponent().getComponentId());
    });
    final Multimap<String, String> issueIdsByConceptIds = HashMultimap.create();
    final Set<String> alreadyFetchedConceptIds = Sets.newHashSet();
    for (ComponentCategory category : issueComponentIdsByComponentCategory.keySet()) {
        final Query<String[]> query = buildQuery(category, issueComponentIdsByComponentCategory.get(category));
        query.stream(searcher).forEachOrdered(hits -> {
            for (String[] hit : hits) {
                String id = hit[0];
                String status = hit[1];
                String moduleId = hit[2];
                issuesByComponentId.get(id).forEach(validationIssue -> {
                    validationIssue.setDetails(COMPONENT_STATUS, status);
                    validationIssue.setDetails(COMPONENT_MODULE_ID, moduleId);
                    if (CONCEPT == category) {
                        validationIssue.setDetails(CONCEPT_STATUS, status);
                        validationIssue.setDetails(SnomedDocument.Fields.EFFECTIVE_TIME, Long.parseLong(hit[3]));
                        alreadyFetchedConceptIds.add(id);
                    } else if (DESCRIPTION == category || RELATIONSHIP == category || SET_MEMBER == category) {
                        validationIssue.setDetails(SnomedDocument.Fields.EFFECTIVE_TIME, Long.parseLong(hit[3]));
                        final String containerConceptId = hit[4];
                        if (!Strings.isNullOrEmpty(containerConceptId) && (!issueIdsByConceptIds.containsKey(containerConceptId) || !alreadyFetchedConceptIds.contains(containerConceptId))) {
                            issueIdsByConceptIds.put(containerConceptId, id);
                        }
                        // in case of description just add the already fetched term as label to the issue, concepts and relationship will get their
                        if (DESCRIPTION == category) {
                            validationIssue.setAffectedComponentLabels(Collections.singletonList(hit[5]));
                        }
                    }
                });
            }
        });
    }
    if (!issueIdsByConceptIds.isEmpty()) {
        final Query<String[]> conceptStatusQuery = Query.select(String[].class).from(SnomedConceptDocument.class).fields(SnomedConceptDocument.Fields.ID, SnomedConceptDocument.Fields.ACTIVE).where(SnomedConceptDocument.Expressions.ids(issueIdsByConceptIds.keySet())).limit(SCROLL_SIZE).build();
        conceptStatusQuery.stream(searcher).flatMap(Hits::stream).forEachOrdered(hit -> {
            Collection<String> issueIds = issueIdsByConceptIds.get(hit[0]);
            issueIds.stream().forEach(id -> {
                issuesByComponentId.get(id).forEach(validationIssue -> validationIssue.setDetails(CONCEPT_STATUS, hit[1]));
            });
        });
    }
}

Example 2 with ValidationIssue

use of com.b2international.snowowl.core.validation.issue.ValidationIssue in project snow-owl by b2ihealthcare.

the class SnomedValidationIssueDetailExtension method extendRelationshipIssueLabels.

private void extendRelationshipIssueLabels(BranchContext context, Collection<ValidationIssue> issues, Map<String, Object> ruleParameters) {
    final RevisionSearcher searcher = context.service(RevisionSearcher.class);
    final List<ValidationIssue> relationshipIssues = issues.stream().filter(issue -> SnomedRelationship.TYPE == issue.getAffectedComponent().getComponentType()).collect(Collectors.toList());
    if (relationshipIssues.isEmpty()) {
        return;
    }
    final Multimap<String, ValidationIssue> issuesByRelationshipId = Multimaps.index(relationshipIssues, issue -> issue.getAffectedComponent().getComponentId());
    final Set<String> conceptsToFetch = newHashSet();
    final Map<String, String> relationshipFragmentsByRelationshipId = Maps.newHashMap();
    searcher.stream(Query.select(String[].class).from(SnomedRelationshipIndexEntry.class).fields(SnomedRelationshipIndexEntry.Fields.ID, SnomedRelationshipIndexEntry.Fields.SOURCE_ID, SnomedRelationshipIndexEntry.Fields.TYPE_ID, SnomedRelationshipIndexEntry.Fields.DESTINATION_ID, SnomedRelationshipIndexEntry.Fields.VALUE_TYPE, SnomedRelationshipIndexEntry.Fields.NUMERIC_VALUE, SnomedRelationshipIndexEntry.Fields.STRING_VALUE).where(SnomedRelationshipIndexEntry.Expressions.ids(issuesByRelationshipId.keySet())).limit(SCROLL_SIZE).build()).forEach(hits -> {
        for (String[] hit : hits) {
            final String id = hit[0];
            final String sourceId = hit[1];
            final String typeId = hit[2];
            final String destinationId = hit[3];
            final String valueType = hit[4];
            final String numericValue = hit[5];
            final String stringValue = hit[6];
            String destination = "";
            conceptsToFetch.add(sourceId);
            conceptsToFetch.add(typeId);
            if (!Strings.isNullOrEmpty(destinationId)) {
                conceptsToFetch.add(destinationId);
                destination = destinationId;
            } else {
                if (RelationshipValueType.DECIMAL.name().equals(valueType) || RelationshipValueType.INTEGER.name().equals(valueType)) {
                    destination = DecimalUtils.decode(numericValue).toString();
                } else if (RelationshipValueType.STRING.name().equals(valueType)) {
                    destination = stringValue;
                }
            }
            relationshipFragmentsByRelationshipId.put(id, String.format("%s|%s|%s", sourceId, typeId, destination));
        }
    });
    Map<String, String> affectedComponentLabelsByConcept = getAffectedComponentLabels(context, ruleParameters, conceptsToFetch);
    if (!affectedComponentLabelsByConcept.isEmpty()) {
        issuesByRelationshipId.values().forEach(issue -> {
            final String[] relationshipFragments = relationshipFragmentsByRelationshipId.get(issue.getAffectedComponent().getComponentId()).split("[|]");
            final String sourceId = relationshipFragments[0];
            final String typeId = relationshipFragments[1];
            final String destinationIdOrValue = relationshipFragments[2];
            final String sourceTerm = affectedComponentLabelsByConcept.getOrDefault(sourceId, sourceId);
            final String typeTerm = affectedComponentLabelsByConcept.getOrDefault(typeId, typeId);
            final String destinationTerm = affectedComponentLabelsByConcept.getOrDefault(destinationIdOrValue, destinationIdOrValue);
            issue.setAffectedComponentLabels(ImmutableList.of(String.format("%s - %s - %s", sourceTerm, typeTerm, destinationTerm)));
        });
    }
}

Example 3 with ValidationIssue

use of com.b2international.snowowl.core.validation.issue.ValidationIssue in project snow-owl by b2ihealthcare.

the class ValidateRequest method doValidate.

private ValidationResult doValidate(BranchContext context, Writer index) throws IOException {
    final String branchPath = context.path();
    ValidationRuleSearchRequestBuilder req = ValidationRequests.rules().prepareSearch();
    TerminologyResource resource = context.service(TerminologyResource.class);
    ResourceURI resourceURI = resource.getResourceURI(branchPath);
    if (!CompareUtils.isEmpty(ruleIds)) {
        req.filterByIds(ruleIds);
    }
    final ValidationRules rules = req.all().build().execute(context);
    final ValidationThreadPool pool = context.service(ValidationThreadPool.class);
    final BlockingQueue<IssuesToPersist> issuesToPersistQueue = Queues.newLinkedBlockingDeque();
    final List<Promise<Object>> validationPromises = Lists.newArrayList();
    // evaluate selected rules
    for (ValidationRule rule : rules) {
        checkArgument(rule.getCheckType() != null, "CheckType is missing for rule " + rule.getId());
        final ValidationRuleEvaluator evaluator = ValidationRuleEvaluator.Registry.get(rule.getType());
        if (evaluator != null) {
            validationPromises.add(pool.submit(rule.getCheckType(), () -> {
                Stopwatch w = Stopwatch.createStarted();
                try {
                    LOG.info("Executing rule '{}'...", rule.getId());
                    final List<?> evaluationResponse = evaluator.eval(context, rule, ruleParameters);
                    issuesToPersistQueue.offer(new IssuesToPersist(rule.getId(), evaluationResponse));
                    LOG.info("Execution of rule '{}' successfully completed in '{}'.", rule.getId(), w);
                // TODO report successfully executed validation rule
                } catch (Exception e) {
                    // TODO report failed validation rule
                    LOG.error("Execution of rule '{}' failed after '{}'.", rule.getId(), w, e);
                }
            }));
        }
    }
    final Set<String> ruleIds = rules.stream().map(ValidationRule::getId).collect(Collectors.toSet());
    final Multimap<String, ComponentIdentifier> whiteListedEntries = fetchWhiteListEntries(context, ruleIds);
    final Promise<List<Object>> promise = Promise.all(validationPromises);
    while (!promise.isDone() || !issuesToPersistQueue.isEmpty()) {
        if (!issuesToPersistQueue.isEmpty()) {
            final Collection<IssuesToPersist> issuesToPersist = newArrayList();
            issuesToPersistQueue.drainTo(issuesToPersist);
            if (!issuesToPersist.isEmpty()) {
                final List<String> rulesToPersist = issuesToPersist.stream().map(itp -> itp.ruleId).collect(Collectors.toList());
                LOG.info("Persisting issues generated by rules '{}'...", rulesToPersist);
                // persist new issues generated by rules so far, extending them using the Issue Extension API
                int persistedIssues = 0;
                final Multimap<String, ValidationIssue> issuesToExtendWithDetailsByToolingId = HashMultimap.create();
                for (IssuesToPersist ruleIssues : Iterables.consumingIterable(issuesToPersist)) {
                    final String ruleId = ruleIssues.ruleId;
                    final List<ValidationIssue> existingRuleIssues = ValidationRequests.issues().prepareSearch().all().filterByResourceUri(resourceURI).filterByRule(ruleId).build().execute(context).getItems();
                    final Set<String> issueIdsToDelete = Sets.newHashSet();
                    final Map<ComponentIdentifier, ValidationIssue> existingIsssuesByComponentIdentifier = new HashMap<>();
                    for (ValidationIssue issue : existingRuleIssues) {
                        if (existingIsssuesByComponentIdentifier.containsKey(issue.getAffectedComponent())) {
                            issueIdsToDelete.add(issue.getId());
                        } else {
                            existingIsssuesByComponentIdentifier.put(issue.getAffectedComponent(), issue);
                        }
                    }
                    // remove all processed whitelist entries
                    final Collection<ComponentIdentifier> ruleWhiteListEntries = whiteListedEntries.removeAll(ruleId);
                    final String toolingId = rules.stream().filter(rule -> ruleId.equals(rule.getId())).findFirst().get().getToolingId();
                    for (ValidationIssueDetails issueDetails : ruleIssues.issueDetails) {
                        final ValidationIssue validationIssue;
                        ComponentIdentifier componentIdentifier = issueDetails.affectedComponentId;
                        if (!existingIsssuesByComponentIdentifier.containsKey(componentIdentifier)) {
                            validationIssue = new ValidationIssue(UUID.randomUUID().toString(), ruleId, ComponentURI.of(resourceURI, componentIdentifier), ruleWhiteListEntries.contains(componentIdentifier));
                        } else {
                            final ValidationIssue issueToCopy = existingIsssuesByComponentIdentifier.get(componentIdentifier);
                            validationIssue = new ValidationIssue(issueToCopy.getId(), issueToCopy.getRuleId(), ComponentURI.of(resourceURI, issueToCopy.getAffectedComponent()), ruleWhiteListEntries.contains(issueToCopy.getAffectedComponent()));
                            existingIsssuesByComponentIdentifier.remove(componentIdentifier);
                        }
                        validationIssue.setDetails(ValidationIssueDetails.HIGHLIGHT_DETAILS, issueDetails.stylingDetails);
                        issuesToExtendWithDetailsByToolingId.put(toolingId, validationIssue);
                        persistedIssues++;
                    }
                    existingRuleIssues.stream().filter(issue -> existingIsssuesByComponentIdentifier.containsKey(issue.getAffectedComponent())).forEach(issue -> issueIdsToDelete.add(issue.getId()));
                    if (!issueIdsToDelete.isEmpty()) {
                        index.removeAll(Collections.singletonMap(ValidationIssue.class, issueIdsToDelete));
                    }
                }
                for (String toolingId : issuesToExtendWithDetailsByToolingId.keySet()) {
                    final ValidationIssueDetailExtension extensions = context.service(ValidationIssueDetailExtensionProvider.class).getExtensions(toolingId);
                    final Collection<ValidationIssue> issues = issuesToExtendWithDetailsByToolingId.removeAll(toolingId);
                    extensions.extendIssues(context, issues, ruleParameters);
                    for (ValidationIssue issue : issues) {
                        index.put(issue);
                    }
                }
                index.commit();
                LOG.info("Persisted '{}' issues generated by rules '{}'.", persistedIssues, rulesToPersist);
            }
        } else {
            try {
                // wait at least number of rules * 50ms for the next responses
                Thread.sleep(Math.min(((long) ruleIds.size()) * 100, POLL_INTERVAL_MAX));
            } catch (InterruptedException e) {
                throw new SnowowlRuntimeException(e);
            }
        }
    }
    // TODO return ValidationResult object with status and new issue IDs as set
    return new ValidationResult(context.info().id(), context.path());
}

Example 4 with ValidationIssue

use of com.b2international.snowowl.core.validation.issue.ValidationIssue in project snow-owl by b2ihealthcare.

the class SnomedValidationIssueDetailTest method duplicateIssueWithSameComponentIdTest.

@Test
public void duplicateIssueWithSameComponentIdTest() throws Exception {
    final String conceptId = RandomSnomedIdentiferGenerator.generateConceptId();
    final ValidationIssue existingIssue = createIssue(conceptId, Collections.emptyMap());
    final ValidationIssue existingIssue2 = createIssue(conceptId, Collections.emptyMap());
    save(existingIssue);
    save(existingIssue2);
    ImmutableMap<String, Object> ruleQuery = ImmutableMap.<String, Object>builder().put("componentType", "concept").put("active", true).build();
    SnomedConceptDocument theConcept = concept(conceptId).active(true).build();
    indexRevision(MAIN, theConcept);
    createSnomedQueryRule(ruleQuery);
    final ValidationIssues issues = validate();
    assertThat(issues.getItems().size()).isEqualTo(1);
}

Example 5 with ValidationIssue

use of com.b2international.snowowl.core.validation.issue.ValidationIssue in project snow-owl by b2ihealthcare.

the class GenericValidationRuleTest method rule38b.

@Test
public void rule38b() throws Exception {
    // Active concepts should have at least one active stated parent
    final String ruleId = "38b";
    indexRule(ruleId);
    final SnomedConceptDocument activeConceptWithStatedParent = concept(generateConceptId()).active(true).statedParents(Long.valueOf(Concepts.MODULE_SCT_MODEL_COMPONENT)).build();
    final SnomedConceptDocument activeConceptWithoutStatedParent = concept(generateConceptId()).active(true).build();
    final SnomedConceptDocument inactiveConceptWithoutStatedParent = concept(generateConceptId()).active(false).build();
    indexRevision(MAIN, activeConceptWithStatedParent, activeConceptWithoutStatedParent, inactiveConceptWithoutStatedParent);
    ValidationIssues issues = validate(ruleId);
    assertThat(issues.stream().map(ValidationIssue::getAffectedComponent).collect(Collectors.toSet())).contains(ComponentIdentifier.of(SnomedConcept.TYPE, activeConceptWithoutStatedParent.getId())).doesNotContainAnyElementsOf(ImmutableList.of(ComponentIdentifier.of(SnomedConcept.TYPE, activeConceptWithStatedParent.getId()), ComponentIdentifier.of(SnomedConcept.TYPE, inactiveConceptWithoutStatedParent.getId())));
}