use of com.b2international.collections.longs.LongSet in project snow-owl by b2ihealthcare.
the class NodeGraph method getAncestors.
public LongSet getAncestors(long conceptId) {
if (!parentMap.keySet().contains(conceptId)) {
return LongCollections.emptySet();
}
LongSet ancestorIds = PrimitiveSets.newLongOpenHashSet();
addAncestorIds(conceptId, ancestorIds);
ancestorIds.remove(conceptId);
return ancestorIds;
}
use of com.b2international.collections.longs.LongSet in project snow-owl by b2ihealthcare.
the class TaxonomyGraph method processElements.
private LongSet processElements(final long conceptId, final BitSet bitSet) {
if (CompareUtils.isEmpty(bitSet)) {
return PrimitiveSets.newLongOpenHashSet();
}
final int count = bitSet.cardinality();
final LongSet $ = PrimitiveSets.newLongOpenHashSetWithExpectedSize(count);
for (int i = bitSet.nextSetBit(0); i >= 0; i = bitSet.nextSetBit(i + 1)) {
long convertedId = getNodeId(i);
if (convertedId == conceptId && checkCycles) {
throw new CycleDetectedException("Concept " + conceptId + " would introduce a cycle in the ISA graph (loop).");
}
$.add(convertedId);
}
return $;
}
use of com.b2international.collections.longs.LongSet in project snow-owl by b2ihealthcare.
the class NormalFormGenerator method computeChanges.
@Override
public final void computeChanges(final IProgressMonitor monitor, final OntologyChangeProcessor<StatementFragment> statementProcessor, final OntologyChangeProcessor<ConcreteDomainFragment> concreteDomainProcessor) {
final Stopwatch stopwatch = Stopwatch.createStarted();
LOGGER.info(">>> Distribution normal form generation");
final LongList entries = reasonerTaxonomy.getIterationOrder();
final SubMonitor subMonitor = SubMonitor.convert(monitor, "Generating distribution normal form...", entries.size() * 2);
try {
LongSet previousLayer = null;
LongSet currentLayer = PrimitiveSets.newLongOpenHashSet();
final Set<Long> graphTypeIds = reasonerTaxonomy.getPropertyChains().stream().map(PropertyChain::getDestinationType).collect(Collectors.toSet());
// The first round can be skipped entirely, if no type IDs participate in a property chain
final boolean propertyChainsPresent = !graphTypeIds.isEmpty();
if (propertyChainsPresent) {
// Initialize node graphs for properties we need to traverse
LOGGER.info("--- Initializing node graphs for types: {}", graphTypeIds);
graphTypeIds.forEach(id -> transitiveNodeGraphs.put(id, new NodeGraph()));
// Round 1: build alternative hierarchies
for (final LongIterator itr = entries.iterator(); itr.hasNext(); ) /* empty */
{
final long conceptId = itr.next();
if (conceptId == ReasonerTaxonomyInferrer.DEPTH_CHANGE) {
if (previousLayer != null) {
invalidate(previousLayer);
}
previousLayer = currentLayer;
currentLayer = PrimitiveSets.newLongOpenHashSet();
continue;
}
precomputeProperties(conceptId, false);
final Collection<StatementFragment> inferredNonIsAFragments = statementCache.get(conceptId);
inferredNonIsAFragments.stream().filter(r -> transitiveNodeGraphs.keySet().contains(r.getTypeId())).filter(StatementFragmentWithDestination.class::isInstance).map(StatementFragmentWithDestination.class::cast).forEachOrdered(r -> transitiveNodeGraphs.get(r.getTypeId()).addParent(conceptId, r.getDestinationId()));
}
// Clear the last layer of concepts
previousLayer = null;
currentLayer = PrimitiveSets.newLongOpenHashSet();
statementCache.clear();
concreteDomainCache.clear();
} else {
LOGGER.info("--- Node graphs computation skipped, no types used for property chaining");
}
LOGGER.info("--- Use node graphs for hierarchy computation");
// Round 2: record changes using the hierarchies
for (final LongIterator itr = entries.iterator(); itr.hasNext(); ) /* empty */
{
final long conceptId = itr.next();
if (conceptId == ReasonerTaxonomyInferrer.DEPTH_CHANGE) {
if (previousLayer != null) {
invalidate(previousLayer);
}
previousLayer = currentLayer;
currentLayer = PrimitiveSets.newLongOpenHashSet();
continue;
}
// Run costly comparison of property chain hierarchies only if there are any
precomputeProperties(conceptId, propertyChainsPresent);
final Collection<StatementFragment> existingStatements = reasonerTaxonomy.getExistingInferredRelationships().get(conceptId);
final Collection<StatementFragment> targetStatements = getTargetRelationships(conceptId);
statementProcessor.apply(conceptId, existingStatements, targetStatements, StatementFragmentOrdering.INSTANCE, subMonitor.newChild(1));
final Collection<ConcreteDomainFragment> existingMembers = reasonerTaxonomy.getInferredConcreteDomainMembers().get(Long.toString(conceptId));
final Collection<ConcreteDomainFragment> targetMembers = getTargetMembers(conceptId);
concreteDomainProcessor.apply(conceptId, existingMembers, targetMembers, ConcreteDomainChangeOrdering.INSTANCE, subMonitor.newChild(1));
}
} finally {
subMonitor.done();
LOGGER.info("<<< Distribution normal form generation [{}]", stopwatch.toString());
}
}
use of com.b2international.collections.longs.LongSet in project snow-owl by b2ihealthcare.
the class SnomedRepositoryPreCommitHook method getChangeSetProcessors.
@Override
protected Collection<ChangeSetProcessor> getChangeSetProcessors(StagingArea staging, RevisionSearcher index) throws IOException {
final RepositoryContext context = ClassUtils.checkAndCast(staging.getContext(), RepositoryContext.class);
// initialize OWL Expression converter on the current branch
final SnomedOWLExpressionConverter expressionConverter = new BranchRequest<>(staging.getBranchPath(), branchContext -> {
return new SnomedOWLExpressionConverter(branchContext.inject().bind(RevisionSearcher.class, index).build());
}).execute(context);
final Set<String> statedSourceIds = Sets.newHashSet();
final Set<String> statedDestinationIds = Sets.newHashSet();
final Set<String> inferredSourceIds = Sets.newHashSet();
final Set<String> inferredDestinationIds = Sets.newHashSet();
collectIds(statedSourceIds, statedDestinationIds, staging.getNewObjects(SnomedRelationshipIndexEntry.class), Concepts.STATED_RELATIONSHIP);
collectIds(statedSourceIds, statedDestinationIds, staging.getChangedRevisions(SnomedRelationshipIndexEntry.class).map(diff -> (SnomedRelationshipIndexEntry) diff.newRevision), Concepts.STATED_RELATIONSHIP);
collectIds(inferredSourceIds, inferredDestinationIds, staging.getNewObjects(SnomedRelationshipIndexEntry.class), Concepts.INFERRED_RELATIONSHIP);
collectIds(inferredSourceIds, inferredDestinationIds, staging.getChangedRevisions(SnomedRelationshipIndexEntry.class).map(diff -> (SnomedRelationshipIndexEntry) diff.newRevision), Concepts.INFERRED_RELATIONSHIP);
collectIds(statedSourceIds, statedDestinationIds, staging.getNewObjects(SnomedRefSetMemberIndexEntry.class), expressionConverter);
collectIds(statedSourceIds, statedDestinationIds, staging.getChangedRevisions(SnomedRefSetMemberIndexEntry.class).map(diff -> (SnomedRefSetMemberIndexEntry) diff.newRevision), expressionConverter);
staging.getRemovedObjects(SnomedRelationshipIndexEntry.class).filter(detachedRelationship -> Concepts.IS_A.equals(detachedRelationship.getTypeId())).forEach(detachedRelationship -> {
// XXX: IS A relationships are expected to have a destination ID, not a value
checkState(!detachedRelationship.hasValue(), "IS A relationship found with value: %s", detachedRelationship.getId());
if (Concepts.STATED_RELATIONSHIP.equals(detachedRelationship.getCharacteristicTypeId())) {
statedSourceIds.add(detachedRelationship.getSourceId());
statedDestinationIds.add(detachedRelationship.getDestinationId());
} else if (Concepts.INFERRED_RELATIONSHIP.equals(detachedRelationship.getCharacteristicTypeId())) {
inferredSourceIds.add(detachedRelationship.getSourceId());
inferredDestinationIds.add(detachedRelationship.getDestinationId());
}
});
staging.getRemovedObjects(SnomedRefSetMemberIndexEntry.class).filter(detachedMember -> SnomedRefSetType.OWL_AXIOM == detachedMember.getReferenceSetType()).forEach(detachedOwlMember -> {
collectIds(statedSourceIds, statedDestinationIds, detachedOwlMember, expressionConverter);
});
final LongSet statedConceptIds = PrimitiveSets.newLongOpenHashSet();
final LongSet inferredConceptIds = PrimitiveSets.newLongOpenHashSet();
if (!statedDestinationIds.isEmpty()) {
for (SnomedConceptDocument statedDestinationConcept : index.get(SnomedConceptDocument.class, statedDestinationIds)) {
statedConceptIds.add(Long.parseLong(statedDestinationConcept.getId()));
if (statedDestinationConcept.getStatedParents() != null) {
statedConceptIds.addAll(statedDestinationConcept.getStatedParents());
}
if (statedDestinationConcept.getStatedAncestors() != null) {
statedConceptIds.addAll(statedDestinationConcept.getStatedAncestors());
}
}
}
if (!inferredDestinationIds.isEmpty()) {
for (SnomedConceptDocument inferredDestinationConcept : index.get(SnomedConceptDocument.class, inferredDestinationIds)) {
inferredConceptIds.add(Long.parseLong(inferredDestinationConcept.getId()));
if (inferredDestinationConcept.getParents() != null) {
inferredConceptIds.addAll(inferredDestinationConcept.getParents());
}
if (inferredDestinationConcept.getAncestors() != null) {
inferredConceptIds.addAll(inferredDestinationConcept.getAncestors());
}
}
}
staging.getRemovedObjects(SnomedDescriptionIndexEntry.class).forEach(removedDescription -> {
if (removedDescription.isFsn() && removedDescription.isActive()) {
statedSourceIds.add(removedDescription.getConceptId());
inferredSourceIds.add(removedDescription.getConceptId());
}
});
staging.getChangedRevisions(SnomedDescriptionIndexEntry.class).filter(diff -> ((SnomedDescriptionIndexEntry) diff.newRevision).isFsn()).filter(diff -> diff.hasRevisionPropertyChanges(ACTIVE_AND_TERM_FIELDS)).forEach(diff -> {
SnomedDescriptionIndexEntry newRevision = (SnomedDescriptionIndexEntry) diff.newRevision;
statedSourceIds.add(newRevision.getConceptId());
inferredSourceIds.add(newRevision.getConceptId());
});
staging.getNewObjects(SnomedDescriptionIndexEntry.class).filter(newDescription -> newDescription.isFsn() && newDescription.isActive()).forEach(newDescription -> {
statedSourceIds.add(newDescription.getConceptId());
inferredSourceIds.add(newDescription.getConceptId());
});
if (!statedSourceIds.isEmpty()) {
final Query<SnomedConceptDocument> statedSourceConceptsQuery = Query.select(SnomedConceptDocument.class).where(Expressions.builder().should(SnomedConceptDocument.Expressions.ids(statedSourceIds)).should(SnomedConceptDocument.Expressions.statedParents(statedSourceIds)).should(SnomedConceptDocument.Expressions.statedAncestors(statedSourceIds)).build()).limit(Integer.MAX_VALUE).build();
for (SnomedConceptDocument statedSourceConcept : index.search(statedSourceConceptsQuery)) {
statedConceptIds.add(Long.parseLong(statedSourceConcept.getId()));
if (statedSourceConcept.getStatedParents() != null) {
statedConceptIds.addAll(statedSourceConcept.getStatedParents());
}
if (statedSourceConcept.getStatedAncestors() != null) {
statedConceptIds.addAll(statedSourceConcept.getStatedAncestors());
}
}
}
if (!inferredSourceIds.isEmpty()) {
final Query<SnomedConceptDocument> inferredSourceConceptsQuery = Query.select(SnomedConceptDocument.class).where(Expressions.builder().should(SnomedConceptDocument.Expressions.ids(inferredSourceIds)).should(SnomedConceptDocument.Expressions.parents(inferredSourceIds)).should(SnomedConceptDocument.Expressions.ancestors(inferredSourceIds)).build()).limit(Integer.MAX_VALUE).build();
for (SnomedConceptDocument inferredSourceConcept : index.search(inferredSourceConceptsQuery)) {
inferredConceptIds.add(Long.parseLong(inferredSourceConcept.getId()));
if (inferredSourceConcept.getParents() != null) {
inferredConceptIds.addAll(inferredSourceConcept.getParents());
}
if (inferredSourceConcept.getAncestors() != null) {
inferredConceptIds.addAll(inferredSourceConcept.getAncestors());
}
}
}
staging.getNewObjects(SnomedConceptDocument.class).forEach(newConcept -> {
long longId = Long.parseLong(newConcept.getId());
statedConceptIds.add(longId);
inferredConceptIds.add(longId);
});
// collect all reactivated concepts for the taxonomy to properly re-register them in the tree even if they don't carry stated/inferred information in this commit, but they have something in the index
staging.getChangedRevisions(SnomedConceptDocument.class, Set.of(SnomedRf2Headers.FIELD_ACTIVE)).forEach(diff -> {
RevisionPropertyDiff propertyDiff = diff.getRevisionPropertyDiff(SnomedRf2Headers.FIELD_ACTIVE);
if ("false".equals(propertyDiff.getOldValue()) && "true".equals(propertyDiff.getNewValue())) {
long longId = Long.parseLong(diff.newRevision.getId());
statedConceptIds.add(longId);
inferredConceptIds.add(longId);
}
});
log.trace("Retrieving taxonomic information from store...");
final boolean checkCycles = !(context instanceof Rf2TransactionContext);
final Taxonomy inferredTaxonomy = Taxonomies.inferred(index, expressionConverter, staging, inferredConceptIds, checkCycles);
final Taxonomy statedTaxonomy = Taxonomies.stated(index, expressionConverter, staging, statedConceptIds, checkCycles);
// XXX change processor execution order is important!!!
return List.of(// those values will be used in the ConceptChangeProcessor for example to properly compute the preferredDescriptions derived field
new DescriptionChangeProcessor(), new ConceptChangeProcessor(DoiDataProvider.INSTANCE, SnomedIconProvider.INSTANCE.getAvailableIconIds(), statedTaxonomy, inferredTaxonomy), new RelationshipChangeProcessor());
}
use of com.b2international.collections.longs.LongSet in project snow-owl by b2ihealthcare.
the class NormalFormGenerator method precomputeProperties.
private void precomputeProperties(final long conceptId, final boolean useNodeGraphs) {
final LongSet parentIds = reasonerTaxonomy.getInferredAncestors().getDestinations(conceptId, true);
/*
* Non IS-A relationships are fetched from ancestors; redundancy must be removed. Since we are working through the list
* of concepts in breadth-first order, we only need to look at cached results from the direct parents, and "distill"
* a non-redundant set of components out of them.
*/
final LongKeyMap<Collection<StatementFragment>> candidateNonIsARelationships = PrimitiveMaps.newLongKeyOpenHashMap();
for (final LongIterator itr = parentIds.iterator(); itr.hasNext(); ) /* empty */
{
final long parentId = itr.next();
candidateNonIsARelationships.put(parentId, statementCache.get(parentId));
}
// Stated axiom fragments are non-IS A, but any stated relationships need to be filtered (if they are still present)
final Collection<StatementFragment> ownStatedRelationships = reasonerTaxonomy.getStatedRelationships().get(conceptId);
final Collection<StatementFragment> ownStatedNonIsaRelationships = ownStatedRelationships.stream().filter(r -> r.getTypeId() != IS_A).collect(Collectors.toList());
candidateNonIsARelationships.put(conceptId, ImmutableList.<StatementFragment>builder().addAll(ownStatedNonIsaRelationships).addAll(reasonerTaxonomy.getAxiomNonIsARelationships().get(conceptId)).addAll(reasonerTaxonomy.getAdditionalGroupedRelationships().get(conceptId)).build());
// Collect existing inferred relationships for cross-referencing group numbers
final Collection<StatementFragment> ownInferredRelationships = reasonerTaxonomy.getExistingInferredRelationships().get(conceptId);
final Collection<StatementFragment> ownInferredNonIsaRelationships = ownInferredRelationships.stream().filter(r -> r.getTypeId() != IS_A).collect(Collectors.toList());
/*
* Do the same as the above, but for CD members
*/
final LongKeyMap<Collection<ConcreteDomainFragment>> candidateMembers = PrimitiveMaps.newLongKeyOpenHashMap();
for (final LongIterator itr = parentIds.iterator(); itr.hasNext(); ) /* empty */
{
final long parentId = itr.next();
candidateMembers.put(parentId, concreteDomainCache.get(parentId));
}
final String referencedComponentId = Long.toString(conceptId);
final Collection<ConcreteDomainFragment> ownStatedMembers = reasonerTaxonomy.getStatedConcreteDomainMembers().get(referencedComponentId);
final Collection<ConcreteDomainFragment> ownAdditionalGroupedMembers = reasonerTaxonomy.getAdditionalGroupedConcreteDomainMembers().get(referencedComponentId);
candidateMembers.put(conceptId, ImmutableList.<ConcreteDomainFragment>builder().addAll(ownStatedMembers).addAll(ownAdditionalGroupedMembers).build());
final Collection<ConcreteDomainFragment> ownInferredMembers = reasonerTaxonomy.getInferredConcreteDomainMembers().get(referencedComponentId);
// Remove redundancy
final NormalFormGroupSet targetGroupSet = getTargetGroupSet(conceptId, parentIds, ownInferredNonIsaRelationships, ownInferredMembers, candidateNonIsARelationships, candidateMembers, useNodeGraphs);
// Extract results; place them in the cache, so following concepts can re-use it
statementCache.put(conceptId, ImmutableList.copyOf(relationshipsFromGroupSet(targetGroupSet)));
concreteDomainCache.put(conceptId, ImmutableList.copyOf(membersFromGroupSet(targetGroupSet)));
}
Aggregations