Search in sources :

Example 1 with TerminologyResource

use of com.b2international.snowowl.core.TerminologyResource in project snow-owl by b2ihealthcare.

the class ValidateRequest method doValidate.

private ValidationResult doValidate(BranchContext context, Writer index) throws IOException {
    final String branchPath = context.path();
    ValidationRuleSearchRequestBuilder req = ValidationRequests.rules().prepareSearch();
    TerminologyResource resource = context.service(TerminologyResource.class);
    ResourceURI resourceURI = resource.getResourceURI(branchPath);
    if (!CompareUtils.isEmpty(ruleIds)) {
        req.filterByIds(ruleIds);
    }
    final ValidationRules rules = req.all().build().execute(context);
    final ValidationThreadPool pool = context.service(ValidationThreadPool.class);
    final BlockingQueue<IssuesToPersist> issuesToPersistQueue = Queues.newLinkedBlockingDeque();
    final List<Promise<Object>> validationPromises = Lists.newArrayList();
    // evaluate selected rules
    for (ValidationRule rule : rules) {
        checkArgument(rule.getCheckType() != null, "CheckType is missing for rule " + rule.getId());
        final ValidationRuleEvaluator evaluator = ValidationRuleEvaluator.Registry.get(rule.getType());
        if (evaluator != null) {
            validationPromises.add(pool.submit(rule.getCheckType(), () -> {
                Stopwatch w = Stopwatch.createStarted();
                try {
                    LOG.info("Executing rule '{}'...", rule.getId());
                    final List<?> evaluationResponse = evaluator.eval(context, rule, ruleParameters);
                    issuesToPersistQueue.offer(new IssuesToPersist(rule.getId(), evaluationResponse));
                    LOG.info("Execution of rule '{}' successfully completed in '{}'.", rule.getId(), w);
                // TODO report successfully executed validation rule
                } catch (Exception e) {
                    // TODO report failed validation rule
                    LOG.error("Execution of rule '{}' failed after '{}'.", rule.getId(), w, e);
                }
            }));
        }
    }
    final Set<String> ruleIds = rules.stream().map(ValidationRule::getId).collect(Collectors.toSet());
    final Multimap<String, ComponentIdentifier> whiteListedEntries = fetchWhiteListEntries(context, ruleIds);
    final Promise<List<Object>> promise = Promise.all(validationPromises);
    while (!promise.isDone() || !issuesToPersistQueue.isEmpty()) {
        if (!issuesToPersistQueue.isEmpty()) {
            final Collection<IssuesToPersist> issuesToPersist = newArrayList();
            issuesToPersistQueue.drainTo(issuesToPersist);
            if (!issuesToPersist.isEmpty()) {
                final List<String> rulesToPersist = issuesToPersist.stream().map(itp -> itp.ruleId).collect(Collectors.toList());
                LOG.info("Persisting issues generated by rules '{}'...", rulesToPersist);
                // persist new issues generated by rules so far, extending them using the Issue Extension API
                int persistedIssues = 0;
                final Multimap<String, ValidationIssue> issuesToExtendWithDetailsByToolingId = HashMultimap.create();
                for (IssuesToPersist ruleIssues : Iterables.consumingIterable(issuesToPersist)) {
                    final String ruleId = ruleIssues.ruleId;
                    final List<ValidationIssue> existingRuleIssues = ValidationRequests.issues().prepareSearch().all().filterByResourceUri(resourceURI).filterByRule(ruleId).build().execute(context).getItems();
                    final Set<String> issueIdsToDelete = Sets.newHashSet();
                    final Map<ComponentIdentifier, ValidationIssue> existingIsssuesByComponentIdentifier = new HashMap<>();
                    for (ValidationIssue issue : existingRuleIssues) {
                        if (existingIsssuesByComponentIdentifier.containsKey(issue.getAffectedComponent())) {
                            issueIdsToDelete.add(issue.getId());
                        } else {
                            existingIsssuesByComponentIdentifier.put(issue.getAffectedComponent(), issue);
                        }
                    }
                    // remove all processed whitelist entries
                    final Collection<ComponentIdentifier> ruleWhiteListEntries = whiteListedEntries.removeAll(ruleId);
                    final String toolingId = rules.stream().filter(rule -> ruleId.equals(rule.getId())).findFirst().get().getToolingId();
                    for (ValidationIssueDetails issueDetails : ruleIssues.issueDetails) {
                        final ValidationIssue validationIssue;
                        ComponentIdentifier componentIdentifier = issueDetails.affectedComponentId;
                        if (!existingIsssuesByComponentIdentifier.containsKey(componentIdentifier)) {
                            validationIssue = new ValidationIssue(UUID.randomUUID().toString(), ruleId, ComponentURI.of(resourceURI, componentIdentifier), ruleWhiteListEntries.contains(componentIdentifier));
                        } else {
                            final ValidationIssue issueToCopy = existingIsssuesByComponentIdentifier.get(componentIdentifier);
                            validationIssue = new ValidationIssue(issueToCopy.getId(), issueToCopy.getRuleId(), ComponentURI.of(resourceURI, issueToCopy.getAffectedComponent()), ruleWhiteListEntries.contains(issueToCopy.getAffectedComponent()));
                            existingIsssuesByComponentIdentifier.remove(componentIdentifier);
                        }
                        validationIssue.setDetails(ValidationIssueDetails.HIGHLIGHT_DETAILS, issueDetails.stylingDetails);
                        issuesToExtendWithDetailsByToolingId.put(toolingId, validationIssue);
                        persistedIssues++;
                    }
                    existingRuleIssues.stream().filter(issue -> existingIsssuesByComponentIdentifier.containsKey(issue.getAffectedComponent())).forEach(issue -> issueIdsToDelete.add(issue.getId()));
                    if (!issueIdsToDelete.isEmpty()) {
                        index.removeAll(Collections.singletonMap(ValidationIssue.class, issueIdsToDelete));
                    }
                }
                for (String toolingId : issuesToExtendWithDetailsByToolingId.keySet()) {
                    final ValidationIssueDetailExtension extensions = context.service(ValidationIssueDetailExtensionProvider.class).getExtensions(toolingId);
                    final Collection<ValidationIssue> issues = issuesToExtendWithDetailsByToolingId.removeAll(toolingId);
                    extensions.extendIssues(context, issues, ruleParameters);
                    for (ValidationIssue issue : issues) {
                        index.put(issue);
                    }
                }
                index.commit();
                LOG.info("Persisted '{}' issues generated by rules '{}'.", persistedIssues, rulesToPersist);
            }
        } else {
            try {
                // wait at least number of rules * 50ms for the next responses
                Thread.sleep(Math.min(((long) ruleIds.size()) * 100, POLL_INTERVAL_MAX));
            } catch (InterruptedException e) {
                throw new SnowowlRuntimeException(e);
            }
        }
    }
    // TODO return ValidationResult object with status and new issue IDs as set
    return new ValidationResult(context.info().id(), context.path());
}
Also used : ValidationIssueDetailExtensionProvider(com.b2international.snowowl.core.validation.issue.ValidationIssueDetailExtensionProvider) java.util(java.util) TerminologyResource(com.b2international.snowowl.core.TerminologyResource) Stopwatch(com.google.common.base.Stopwatch) Promise(com.b2international.snowowl.core.events.util.Promise) LoggerFactory(org.slf4j.LoggerFactory) ValidationIssue(com.b2international.snowowl.core.validation.issue.ValidationIssue) ValidationIssueDetailExtension(com.b2international.snowowl.core.validation.issue.ValidationIssueDetailExtension) Preconditions.checkArgument(com.google.common.base.Preconditions.checkArgument) AccessControl(com.b2international.snowowl.core.authorization.AccessControl) Permission(com.b2international.snowowl.core.identity.Permission) CompareUtils(com.b2international.commons.CompareUtils) ValidationRuleSearchRequestBuilder(com.b2international.snowowl.core.validation.rule.ValidationRuleSearchRequestBuilder) com.google.common.collect(com.google.common.collect) ValidationRepository(com.b2international.snowowl.core.internal.validation.ValidationRepository) ResourceURI(com.b2international.snowowl.core.ResourceURI) ValidationRule(com.b2international.snowowl.core.validation.rule.ValidationRule) ValidationWhiteListSearchRequestBuilder(com.b2international.snowowl.core.validation.whitelist.ValidationWhiteListSearchRequestBuilder) ValidationThreadPool(com.b2international.snowowl.core.internal.validation.ValidationThreadPool) Logger(org.slf4j.Logger) SnowowlRuntimeException(com.b2international.snowowl.core.api.SnowowlRuntimeException) Request(com.b2international.snowowl.core.events.Request) ComponentURI(com.b2international.snowowl.core.uri.ComponentURI) IOException(java.io.IOException) BlockingQueue(java.util.concurrent.BlockingQueue) Collectors(java.util.stream.Collectors) Writer(com.b2international.index.Writer) ValidationRuleEvaluator(com.b2international.snowowl.core.validation.eval.ValidationRuleEvaluator) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) ComponentIdentifier(com.b2international.snowowl.core.ComponentIdentifier) ValidationRules(com.b2international.snowowl.core.validation.rule.ValidationRules) BranchContext(com.b2international.snowowl.core.domain.BranchContext) ValidationRuleSearchRequestBuilder(com.b2international.snowowl.core.validation.rule.ValidationRuleSearchRequestBuilder) ValidationThreadPool(com.b2international.snowowl.core.internal.validation.ValidationThreadPool) Stopwatch(com.google.common.base.Stopwatch) ComponentIdentifier(com.b2international.snowowl.core.ComponentIdentifier) ValidationIssueDetailExtension(com.b2international.snowowl.core.validation.issue.ValidationIssueDetailExtension) TerminologyResource(com.b2international.snowowl.core.TerminologyResource) Lists.newArrayList(com.google.common.collect.Lists.newArrayList) ResourceURI(com.b2international.snowowl.core.ResourceURI) ValidationIssue(com.b2international.snowowl.core.validation.issue.ValidationIssue) SnowowlRuntimeException(com.b2international.snowowl.core.api.SnowowlRuntimeException) IOException(java.io.IOException) SnowowlRuntimeException(com.b2international.snowowl.core.api.SnowowlRuntimeException) ValidationRule(com.b2international.snowowl.core.validation.rule.ValidationRule) Promise(com.b2international.snowowl.core.events.util.Promise) ValidationRules(com.b2international.snowowl.core.validation.rule.ValidationRules) ValidationRuleEvaluator(com.b2international.snowowl.core.validation.eval.ValidationRuleEvaluator) ValidationIssueDetailExtensionProvider(com.b2international.snowowl.core.validation.issue.ValidationIssueDetailExtensionProvider)

Example 2 with TerminologyResource

use of com.b2international.snowowl.core.TerminologyResource in project snow-owl by b2ihealthcare.

the class ClassificationJobRequest method executeClassification.

private void executeClassification(final BranchContext context, final String classificationId, final ClassificationTracker tracker) {
    final RevisionSearcher revisionSearcher = context.service(RevisionSearcher.class);
    TerminologyResource resource = context.service(TerminologyResource.class);
    @SuppressWarnings("unchecked") final Set<String> reasonerExcludedModuleIds = Collections3.toImmutableSet((Iterable) resource.getSettings().getOrDefault(REASONER_EXCLUDE_MODULE_IDS, Collections.emptySet()));
    final SnomedCoreConfiguration configuration = context.service(SnomedCoreConfiguration.class);
    final boolean concreteDomainSupported = configuration.isConcreteDomainSupported();
    final ReasonerTaxonomy taxonomy;
    try (Locks locks = Locks.on(context).lock(DatastoreLockContextDescriptions.CLASSIFY, parentLockContext)) {
        taxonomy = buildTaxonomy(revisionSearcher, reasonerExcludedModuleIds, concreteDomainSupported);
    } catch (final LockedException e) {
        throw new ReasonerApiException("Couldn't acquire exclusive access to terminology store for classification; %s", e.getMessage(), e);
    }
    final OWLOntologyManager ontologyManager = OWLManager.createOWLOntologyManager();
    ontologyManager.getOntologyFactories().add(new DelegateOntologyFactory(taxonomy));
    // TODO: custom moduleId in ontology IRI?
    final IRI ontologyIRI = IRI.create(DelegateOntology.NAMESPACE_SCTM + Concepts.MODULE_SCT_CORE);
    try {
        final DelegateOntology ontology = (DelegateOntology) ontologyManager.createOntology(ontologyIRI);
        final ReasonerTaxonomyInferrer inferrer = new ReasonerTaxonomyInferrer(reasonerId, ontology, context);
        final ReasonerTaxonomy inferredTaxonomy = inferrer.addInferences(taxonomy);
        final NormalFormGenerator normalFormGenerator = new NormalFormGenerator(inferredTaxonomy);
        tracker.classificationCompleted(classificationId, inferredTaxonomy, normalFormGenerator);
    } catch (final OWLOntologyCreationException e) {
        throw new ReasonerApiException("Exception caught while creating ontology instance.", e);
    } catch (final ReasonerInterruptedException | OWLReasonerRuntimeException e) {
        throw new ReasonerApiException("Exception caught while classifying the ontology.", e);
    }
}
Also used : ReasonerTaxonomy(com.b2international.snowowl.snomed.datastore.index.taxonomy.ReasonerTaxonomy) IRI(org.semanticweb.owlapi.model.IRI) LockedException(com.b2international.commons.exceptions.LockedException) ReasonerInterruptedException(org.semanticweb.owlapi.reasoner.ReasonerInterruptedException) SnomedCoreConfiguration(com.b2international.snowowl.snomed.datastore.config.SnomedCoreConfiguration) ReasonerApiException(com.b2international.snowowl.snomed.reasoner.exceptions.ReasonerApiException) DelegateOntologyFactory(com.b2international.snowowl.snomed.reasoner.ontology.DelegateOntologyFactory) Locks(com.b2international.snowowl.core.locks.Locks) NormalFormGenerator(com.b2international.snowowl.snomed.reasoner.normalform.NormalFormGenerator) OWLReasonerRuntimeException(org.semanticweb.owlapi.reasoner.OWLReasonerRuntimeException) DelegateOntology(com.b2international.snowowl.snomed.reasoner.ontology.DelegateOntology) ReasonerTaxonomyInferrer(com.b2international.snowowl.snomed.reasoner.classification.ReasonerTaxonomyInferrer) OWLOntologyCreationException(org.semanticweb.owlapi.model.OWLOntologyCreationException) TerminologyResource(com.b2international.snowowl.core.TerminologyResource) OWLOntologyManager(org.semanticweb.owlapi.model.OWLOntologyManager) RevisionSearcher(com.b2international.index.revision.RevisionSearcher)

Example 3 with TerminologyResource

use of com.b2international.snowowl.core.TerminologyResource in project snow-owl by b2ihealthcare.

the class SaveJobRequest method createNamespaceAndModuleAssigner.

private SnomedNamespaceAndModuleAssigner createNamespaceAndModuleAssigner(final BranchContext context) {
    // Override assigner type if given
    final String selectedType;
    if (assignerType != null) {
        selectedType = assignerType;
    } else {
        final TerminologyResource resource = context.service(TerminologyResource.class);
        selectedType = (String) resource.getSettings().getOrDefault(NAMESPACE_AND_MODULE_ASSIGNER, DEFAULT_NAMESPACE_AND_MODULE_ASSIGNER);
    }
    final SnomedNamespaceAndModuleAssigner assigner = SnomedNamespaceAndModuleAssigner.create(context, selectedType, moduleId, namespace);
    LOG.info("Reasoner service will use {} for relationship/concrete domain namespace and module assignment.", assigner);
    return assigner;
}
Also used : TerminologyResource(com.b2international.snowowl.core.TerminologyResource) SnomedNamespaceAndModuleAssigner(com.b2international.snowowl.snomed.datastore.id.assigner.SnomedNamespaceAndModuleAssigner)

Example 4 with TerminologyResource

use of com.b2international.snowowl.core.TerminologyResource in project snow-owl by b2ihealthcare.

the class OntologyExportRequest method execute.

@Override
public String execute(final BranchContext context) {
    final RevisionSearcher revisionSearcher = context.service(RevisionSearcher.class);
    TerminologyResource resource = context.service(TerminologyResource.class);
    @SuppressWarnings("unchecked") final Set<String> reasonerExcludedModuleIds = Collections3.toImmutableSet((Iterable) resource.getSettings().getOrDefault(REASONER_EXCLUDE_MODULE_IDS, Collections.emptySet()));
    final SnomedCoreConfiguration configuration = context.service(SnomedCoreConfiguration.class);
    final boolean concreteDomainSupportEnabled = configuration.isConcreteDomainSupported();
    final ReasonerTaxonomyBuilder taxonomyBuilder = new ReasonerTaxonomyBuilder(reasonerExcludedModuleIds);
    taxonomyBuilder.addActiveConceptIds(revisionSearcher);
    taxonomyBuilder.finishConcepts();
    taxonomyBuilder.addFullySpecifiedNames(revisionSearcher);
    taxonomyBuilder.addConceptFlags(revisionSearcher);
    taxonomyBuilder.addActiveStatedEdges(revisionSearcher);
    taxonomyBuilder.addActiveStatedRelationships(revisionSearcher);
    taxonomyBuilder.addNeverGroupedTypeIds(revisionSearcher);
    taxonomyBuilder.addActiveAxioms(revisionSearcher);
    if (concreteDomainSupportEnabled) {
        taxonomyBuilder.addActiveConcreteDomainMembers(revisionSearcher);
    }
    final ReasonerTaxonomy taxonomy = taxonomyBuilder.build();
    final OWLOntologyManager ontologyManager = OWLManager.createOWLOntologyManager();
    ontologyManager.getOntologyFactories().add(new DelegateOntologyFactory(taxonomy));
    final IRI ontologyIRI = IRI.create(DelegateOntology.NAMESPACE_SCTM + ontologyModuleId);
    try {
        final OWLOntology ontology = ontologyManager.createOntology(ontologyIRI);
        OWLOntology ontologyToExport = ontologyManager.createOntology();
        ontology.getAxioms().forEach(axiom -> {
            ontologyManager.addAxiom(ontologyToExport, axiom);
        });
        final OWLDocumentFormat documentFormat = getOWLDocumentFormat();
        final AttachmentRegistry fileRegistry = context.service(AttachmentRegistry.class);
        final UUID id = UUID.randomUUID();
        final PipedOutputStream os = new PipedOutputStream();
        final PipedInputStream is = new PipedInputStream(os, PIPE_SIZE);
        final ForkJoinTask<?> uploadTask = ForkJoinTask.adapt(() -> fileRegistry.upload(id, is));
        final ForkJoinTask<?> saveTask = ForkJoinTask.adapt(() -> {
            try {
                ontologyManager.saveOntology(ontologyToExport, documentFormat, os);
            } catch (final OWLOntologyStorageException e) {
                throw createExportFailedException(context, e);
            } finally {
                try {
                    os.close();
                } catch (final IOException e) {
                    throw createExportFailedException(context, e);
                }
            }
        });
        ForkJoinTask.invokeAll(saveTask, uploadTask);
        return id.toString();
    } catch (final OWLOntologyCreationException e) {
        throw createExportFailedException(context, e);
    } catch (final IOException e) {
        throw createExportFailedException(context, e);
    } finally {
        // invalidate static cache entries :'(
        ontologyManager.getOntologies().forEach(o -> {
            ontologyManager.applyChange(new SetOntologyID(o, new OWLOntologyID()));
        });
    }
}
Also used : ReasonerTaxonomy(com.b2international.snowowl.snomed.datastore.index.taxonomy.ReasonerTaxonomy) ReasonerTaxonomyBuilder(com.b2international.snowowl.snomed.datastore.index.taxonomy.ReasonerTaxonomyBuilder) AttachmentRegistry(com.b2international.snowowl.core.attachments.AttachmentRegistry) SnomedCoreConfiguration(com.b2international.snowowl.snomed.datastore.config.SnomedCoreConfiguration) DelegateOntologyFactory(com.b2international.snowowl.snomed.reasoner.ontology.DelegateOntologyFactory) PipedOutputStream(java.io.PipedOutputStream) PipedInputStream(java.io.PipedInputStream) IOException(java.io.IOException) TerminologyResource(com.b2international.snowowl.core.TerminologyResource) UUID(java.util.UUID) RevisionSearcher(com.b2international.index.revision.RevisionSearcher)

Example 5 with TerminologyResource

use of com.b2international.snowowl.core.TerminologyResource in project snow-owl by b2ihealthcare.

the class TerminologyResourceContentRequest method execute.

@Override
public R execute(TerminologyResourceContext context) {
    final ResourceURI resourceURI = context.resourceURI();
    final TerminologyResource resource = context.resource();
    final PathWithVersion branchPathWithVersion = context.service(ResourceURIPathResolver.class).resolveWithVersion(context, resourceURI, resource);
    final String path = branchPathWithVersion.getPath();
    final ResourceURI versionResourceURI = branchPathWithVersion.getVersionResourceURI();
    if (versionResourceURI != null) {
        context = context.inject().bind(ResourceURI.class, versionResourceURI).bind(PathWithVersion.class, branchPathWithVersion).build();
    }
    return new RepositoryRequest<R>(resource.getToolingId(), new BranchRequest<R>(path, next())).execute(context);
}
Also used : ResourceURI(com.b2international.snowowl.core.ResourceURI) ResourceURIPathResolver(com.b2international.snowowl.core.uri.ResourceURIPathResolver) TerminologyResource(com.b2international.snowowl.core.TerminologyResource) PathWithVersion(com.b2international.snowowl.core.uri.ResourceURIPathResolver.PathWithVersion) BranchRequest(com.b2international.snowowl.core.request.BranchRequest)

Aggregations

TerminologyResource (com.b2international.snowowl.core.TerminologyResource)9 ResourceURI (com.b2international.snowowl.core.ResourceURI)6 CompareUtils (com.b2international.commons.CompareUtils)3 BadRequestException (com.b2international.commons.exceptions.BadRequestException)2 RevisionSearcher (com.b2international.index.revision.RevisionSearcher)2 Resource (com.b2international.snowowl.core.Resource)2 SnowowlRuntimeException (com.b2international.snowowl.core.api.SnowowlRuntimeException)2 AttachmentRegistry (com.b2international.snowowl.core.attachments.AttachmentRegistry)2 AccessControl (com.b2international.snowowl.core.authorization.AccessControl)2 Branch (com.b2international.snowowl.core.branch.Branch)2 BranchContext (com.b2international.snowowl.core.domain.BranchContext)2 Request (com.b2international.snowowl.core.events.Request)2 Permission (com.b2international.snowowl.core.identity.Permission)2 ResourceURIPathResolver (com.b2international.snowowl.core.uri.ResourceURIPathResolver)2 VersionDocument (com.b2international.snowowl.core.version.VersionDocument)2 Versions (com.b2international.snowowl.core.version.Versions)2 SnomedCoreConfiguration (com.b2international.snowowl.snomed.datastore.config.SnomedCoreConfiguration)2 ReasonerTaxonomy (com.b2international.snowowl.snomed.datastore.index.taxonomy.ReasonerTaxonomy)2 DelegateOntologyFactory (com.b2international.snowowl.snomed.reasoner.ontology.DelegateOntologyFactory)2 IOException (java.io.IOException)2