Examples with Rf2EffectiveTimeSlice com.b2international.snowowl.snomed.datastore.request.rf2.importer.Rf2EffectiveTimeSlice used on opensource projects

Example 1 with Rf2EffectiveTimeSlice

use of com.b2international.snowowl.snomed.datastore.request.rf2.importer.Rf2EffectiveTimeSlice in project snow-owl by b2ihealthcare.

the class Rf2GlobalValidator method removeSkippableMembers.

private void removeSkippableMembers(List<Rf2EffectiveTimeSlice> slices) {
    for (Rf2EffectiveTimeSlice slice : slices) {
        for (Map.Entry<String, String> entry : skippableMemberDependenciesByEffectiveTime.entrySet()) {
            String effectiveTime = entry.getValue();
            if (!slice.getEffectiveTime().equals(effectiveTime)) {
                // Does not concern this effective time slice
                continue;
            }
            String componentId = entry.getKey();
            if (dependenciesByEffectiveTime.containsKey(componentId)) {
                // Component is required elsewhere as well, should not be skipped
                continue;
            }
            slice.unregisterDependencies(componentId);
        }
    }
}

Example 2 with Rf2EffectiveTimeSlice

use of com.b2international.snowowl.snomed.datastore.request.rf2.importer.Rf2EffectiveTimeSlice in project snow-owl by b2ihealthcare.

the class SnomedRf2ImportRequest method doImport.

ImportResponse doImport(final BranchContext context, final File rf2Archive, final Rf2ImportConfiguration importconfig) throws Exception {
    final ResourceURI codeSystemUri = context.service(ResourceURI.class);
    final Rf2ValidationIssueReporter reporter = new Rf2ValidationIssueReporter();
    String latestVersionEffectiveTime = EffectiveTimes.format(ResourceRequests.prepareSearchVersion().one().filterByResource(codeSystemUri.withoutPath()).sortBy("effectiveTime:desc").buildAsync().execute(context).first().map(Version::getEffectiveTime).orElse(LocalDate.EPOCH), DateFormats.SHORT);
    try (final DB db = createDb()) {
        // Read effective time slices from import files
        final Rf2EffectiveTimeSlices effectiveTimeSlices = new Rf2EffectiveTimeSlices(db, isLoadOnDemandEnabled(), latestVersionEffectiveTime, importUntil == null ? null : EffectiveTimes.format(importUntil, DateFormats.SHORT));
        Stopwatch w = Stopwatch.createStarted();
        read(rf2Archive, effectiveTimeSlices, reporter);
        log.info("Preparing RF2 import took: {}", w);
        w.reset().start();
        // Log issues with rows from the import files
        logValidationIssues(reporter);
        if (reporter.hasErrors()) {
            return ImportResponse.defects(reporter.getDefects());
        }
        // Run validation that takes current terminology content into account
        final List<Rf2EffectiveTimeSlice> orderedEffectiveTimeSlices = effectiveTimeSlices.consumeInOrder();
        final Rf2GlobalValidator globalValidator = new Rf2GlobalValidator(log, ignoreMissingReferencesIn);
        /* 
			 * TODO: Use Attachment to get the release file name and/or track file and line number sources for each row 
			 * so that they can be referenced in this stage as well
			 */
        final ImportDefectAcceptor globalDefectAcceptor = reporter.getDefectAcceptor("RF2 release");
        globalValidator.validateTerminologyComponents(orderedEffectiveTimeSlices, globalDefectAcceptor, context);
        // globalValidator.validateMembers(orderedEffectiveTimeSlices, globalDefectAcceptor, context);
        // Log validation issues (but just the ones found during global validation)
        logValidationIssues(globalDefectAcceptor);
        if (reporter.hasErrors()) {
            return ImportResponse.defects(reporter.getDefects());
        }
        // Import effective time slices in chronological order
        final ImmutableSet.Builder<ComponentURI> visitedComponents = ImmutableSet.builder();
        // if not a dryRun, perform import
        if (!dryRun) {
            // Import effective time slices in chronological order
            for (Rf2EffectiveTimeSlice slice : orderedEffectiveTimeSlices) {
                slice.doImport(context, codeSystemUri, importconfig, visitedComponents);
            }
            // Update locales registered on the code system
            updateCodeSystemSettings(context, codeSystemUri);
        }
        return ImportResponse.success(visitedComponents.build(), reporter.getDefects());
    }
}

Example 3 with Rf2EffectiveTimeSlice

use of com.b2international.snowowl.snomed.datastore.request.rf2.importer.Rf2EffectiveTimeSlice in project snow-owl by b2ihealthcare.

the class Rf2GlobalValidator method validateTerminologyComponents.

public void validateTerminologyComponents(final List<Rf2EffectiveTimeSlice> slices, final ImportDefectAcceptor globalDefectAcceptor, final BranchContext context) {
    dependenciesByEffectiveTime = Maps.newHashMap();
    skippableMemberDependenciesByEffectiveTime = Maps.newHashMap();
    // Check the slices in reverse order for missing dependencies
    for (Rf2EffectiveTimeSlice slice : Lists.reverse(slices)) {
        final String effectiveTimeLabel = Rf2EffectiveTimeSlice.SNAPSHOT_SLICE.equals(slice.getEffectiveTime()) ? "..." : String.format(" in effective time '%s'", slice.getEffectiveTime());
        log.info("Validating component consistency{}", effectiveTimeLabel);
        // Resolve pending dependencies with the current slice content
        final Set<String> contentInSlice = Set.copyOf(slice.getContent().keySet());
        dependenciesByEffectiveTime.keySet().removeAll(contentInSlice);
        // Core component dependencies
        for (final LongSet componentDependencies : slice.getDependenciesByComponent().values()) {
            final LongIterator it = componentDependencies.iterator();
            while (it.hasNext()) {
                final long dependencyId = it.next();
                final String stringDependencyId = Long.toString(dependencyId);
                // Insert or update any entry with the current slice key, unless the current slice has the component
                if (!contentInSlice.contains(stringDependencyId)) {
                    dependenciesByEffectiveTime.put(stringDependencyId, slice.getEffectiveTime());
                }
            }
        }
        // Dependencies of reference set members includes the reference component ID, check them separately
        final LongIterator referencedComponentIds = slice.getMembersByReferencedComponent().keySet().iterator();
        while (referencedComponentIds.hasNext()) {
            final long referencedComponentId = referencedComponentIds.next();
            final String stringReferencedComponentId = Long.toString(referencedComponentId);
            if (!contentInSlice.contains(stringReferencedComponentId)) {
                Set<String> referringMembers = slice.getMembersByReferencedComponent().get(referencedComponentId);
                boolean failOnMissingReferences = false;
                for (String memberId : referringMembers) {
                    String[] referringMember = slice.getContent().get(memberId);
                    String referenceSet = referringMember[5];
                    failOnMissingReferences = !ignoreMissingReferencesIn.contains(referenceSet);
                }
                if (failOnMissingReferences) {
                    dependenciesByEffectiveTime.put(stringReferencedComponentId, slice.getEffectiveTime());
                } else {
                    skippableMemberDependenciesByEffectiveTime.put(stringReferencedComponentId, slice.getEffectiveTime());
                }
            }
        }
        // Validate reference set type in similar reference sets
        validateType(slice, globalDefectAcceptor);
    }
    final Set<String> missingConceptIds = newHashSet();
    final Set<String> missingDescriptionIds = newHashSet();
    final Set<String> missingRelationshipIds = newHashSet();
    // Anything that remains is not resolved by the imported data; check if it is in Snow Owl
    if (!dependenciesByEffectiveTime.isEmpty()) {
        Set<String> conceptIds = newHashSet();
        Set<String> descriptionIds = newHashSet();
        Set<String> relationshipIds = newHashSet();
        dependenciesByEffectiveTime.keySet().stream().forEach(id -> {
            switch(SnomedIdentifiers.getComponentCategory(id)) {
                case CONCEPT:
                    conceptIds.add(id);
                    break;
                case DESCRIPTION:
                    descriptionIds.add(id);
                    break;
                case RELATIONSHIP:
                    relationshipIds.add(id);
                    break;
                default:
                    log.error("Unknown component type for identifier: {}", id);
                    break;
            }
        });
        skippableMemberDependenciesByEffectiveTime.keySet().stream().forEach(id -> {
            switch(SnomedIdentifiers.getComponentCategory(id)) {
                case CONCEPT:
                    conceptIds.add(id);
                    break;
                case DESCRIPTION:
                    descriptionIds.add(id);
                    break;
                case RELATIONSHIP:
                    relationshipIds.add(id);
                    break;
                default:
                    log.error("Unknown component type for identifier: {}", id);
                    break;
            }
        });
        log.trace("Fetch existing concept IDs...");
        missingConceptIds.addAll(fetchComponentIds(context, conceptIds, SnomedConceptDocument.class));
        if (!missingConceptIds.isEmpty()) {
            missingConceptIds.forEach(id -> {
                reportMissingComponent(globalDefectAcceptor, id, dependenciesByEffectiveTime.get(id), "concept", canBeSkipped(id));
            });
        }
        log.trace("Fetch existing description IDs...");
        missingDescriptionIds.addAll(fetchComponentIds(context, descriptionIds, SnomedDescriptionIndexEntry.class));
        if (!missingDescriptionIds.isEmpty()) {
            missingDescriptionIds.forEach(id -> {
                reportMissingComponent(globalDefectAcceptor, id, dependenciesByEffectiveTime.get(id), "description", canBeSkipped(id));
            });
        }
        log.trace("Fetch existing relationship IDs...");
        missingRelationshipIds.addAll(fetchComponentIds(context, relationshipIds, SnomedRelationshipIndexEntry.class));
        if (!missingRelationshipIds.isEmpty()) {
            missingRelationshipIds.forEach(id -> {
                reportMissingComponent(globalDefectAcceptor, id, dependenciesByEffectiveTime.get(id), "relationship", canBeSkipped(id));
            });
        }
    }
    removeSkippableMembers(slices);
    skippableMemberDependenciesByEffectiveTime.clear();
    dependenciesByEffectiveTime.clear();
}