Search in sources :

Example 1 with Mapping

use of com.cerner.bunsen.spark.codes.Mapping in project bunsen by cerner.

the class AbstractConceptMaps method getMappings.

/**
 * Returns a dataset with the mappings for each uri and version.
 *
 * @param uriToVersion a map of concept map URI to the version to load
 * @return a dataset of mappings for the given URIs and versions.
 */
public Dataset<Mapping> getMappings(Map<String, String> uriToVersion) {
    JavaSparkContext context = new JavaSparkContext(this.spark.sparkContext());
    Broadcast<Map<String, String>> broadcastMaps = context.broadcast(uriToVersion);
    return this.mappings.filter((FilterFunction<Mapping>) mapping -> {
        String latestVersion = broadcastMaps.getValue().get(mapping.getConceptMapUri());
        return latestVersion != null && latestVersion.equals(mapping.getConceptMapVersion());
    });
}
Also used : Arrays(java.util.Arrays) SparkRowConverter(com.cerner.bunsen.spark.SparkRowConverter) Dataset(org.apache.spark.sql.Dataset) JavaSparkContext(org.apache.spark.api.java.JavaSparkContext) Mapping(com.cerner.bunsen.spark.codes.Mapping) NoSuchTableException(org.apache.spark.sql.catalyst.analysis.NoSuchTableException) UrlAndVersion(com.cerner.bunsen.spark.codes.UrlAndVersion) FhirVersionEnum(ca.uhn.fhir.context.FhirVersionEnum) IBaseResource(org.hl7.fhir.instance.model.api.IBaseResource) Map(java.util.Map) org.apache.spark.sql.functions.lit(org.apache.spark.sql.functions.lit) IParser(ca.uhn.fhir.parser.IParser) BroadcastableMappings(com.cerner.bunsen.spark.codes.broadcast.BroadcastableMappings) JavaRDD(org.apache.spark.api.java.JavaRDD) SparkSession(org.apache.spark.sql.SparkSession) Broadcast(org.apache.spark.broadcast.Broadcast) SaveMode(org.apache.spark.sql.SaveMode) Timestamp(java.sql.Timestamp) Set(java.util.Set) IOException(java.io.IOException) Row(org.apache.spark.sql.Row) Tuple2(scala.Tuple2) Collectors(java.util.stream.Collectors) Encoders(org.apache.spark.sql.Encoders) List(java.util.List) Encoder(org.apache.spark.sql.Encoder) org.apache.spark.sql.functions.col(org.apache.spark.sql.functions.col) FhirContexts(com.cerner.bunsen.FhirContexts) Pattern(java.util.regex.Pattern) Function(org.apache.spark.api.java.function.Function) org.apache.spark.sql.functions(org.apache.spark.sql.functions) FilterFunction(org.apache.spark.api.java.function.FilterFunction) Mapping(com.cerner.bunsen.spark.codes.Mapping) JavaSparkContext(org.apache.spark.api.java.JavaSparkContext) Map(java.util.Map)

Example 2 with Mapping

use of com.cerner.bunsen.spark.codes.Mapping in project bunsen by cerner.

the class ConceptMaps method addToConceptMap.

@Override
protected void addToConceptMap(ConceptMap map, Dataset<Mapping> mappings) {
    // Sort the items so they are grouped together optimally, and so
    // we consistently produce the same ordering, therefore making
    // inspection and comparison of the concept maps easier.
    List<Mapping> sortedMappings = mappings.sort("sourceSystem", "targetSystem", "sourceValue", "targetValue").collectAsList();
    ConceptMapGroupComponent currentGroup = null;
    SourceElementComponent element = null;
    // Workaround for the decoder producing an immutable array by
    // replacing it with a mutable one.
    map.setGroup(new ArrayList<>(map.getGroup()));
    for (Mapping mapping : sortedMappings) {
        // Add a new group if we don't match the previous one.
        if (currentGroup == null || !mapping.getSourceSystem().equals(currentGroup.getSource()) || !mapping.getTargetSystem().equals(currentGroup.getTarget())) {
            currentGroup = null;
            // Find a matching group.
            for (ConceptMapGroupComponent candidate : map.getGroup()) {
                if (mapping.getSourceSystem().equals(candidate.getSource()) && mapping.getTargetSystem().equals(candidate.getTarget())) {
                    currentGroup = candidate;
                    // Workaround for the decoder producing an immutable array by
                    // replacing it with a mutable one.
                    currentGroup.setElement(new ArrayList<>(currentGroup.getElement()));
                    break;
                }
            }
            // No matching group found, so add it.
            if (currentGroup == null) {
                currentGroup = map.addGroup();
                currentGroup.setSource(mapping.getSourceSystem());
                currentGroup.setTarget(mapping.getTargetSystem());
                // Ensure a new element is created for the newly created group.
                element = null;
            }
        }
        // so add one if it does not match the previous.
        if (element == null || !mapping.getSourceValue().equals(element.getCode())) {
            element = currentGroup.addElement();
            element.setCode(mapping.getSourceValue());
        }
        element.addTarget().setCode(mapping.getTargetValue());
    }
}
Also used : Mapping(com.cerner.bunsen.spark.codes.Mapping) ConceptMapGroupComponent(org.hl7.fhir.dstu3.model.ConceptMap.ConceptMapGroupComponent) SourceElementComponent(org.hl7.fhir.dstu3.model.ConceptMap.SourceElementComponent)

Example 3 with Mapping

use of com.cerner.bunsen.spark.codes.Mapping in project bunsen by cerner.

the class ConceptMapsTest method testExpandMappings.

@Test
public void testExpandMappings() {
    ConceptMap conceptMap = ConceptMaps.getEmpty(spark).withConceptMaps(conceptMap("urn:cerner:conceptmap:map", "1")).getConceptMap("urn:cerner:conceptmap:map", "1");
    List<Mapping> mappings = ConceptMaps.expandMappings(conceptMap);
    Mapping expectedValue = new Mapping("urn:cerner:conceptmap:map", "1", "urn:source:valueset", "urn:target:valueset", "urn:source:system", "urn:source:code:a", "urn:target:system", "urn:target:code:1", Mapping.EQUIVALENT);
    Assert.assertEquals(1, mappings.size());
    Assert.assertEquals(expectedValue, mappings.get(0));
}
Also used : Mapping(com.cerner.bunsen.spark.codes.Mapping) ConceptMap(org.hl7.fhir.dstu3.model.ConceptMap) Test(org.junit.Test)

Example 4 with Mapping

use of com.cerner.bunsen.spark.codes.Mapping in project bunsen by cerner.

the class ConceptMaps method expandMappingsIterator.

private static Iterator<Mapping> expandMappingsIterator(ConceptMap map) {
    List<Mapping> mappings = new ArrayList<>();
    for (ConceptMapGroupComponent group : map.getGroup()) {
        for (SourceElementComponent element : group.getElement()) {
            for (TargetElementComponent target : element.getTarget()) {
                Mapping mapping = new Mapping();
                mapping.setConceptMapUri(map.getUrl());
                mapping.setConceptMapVersion(map.getVersion());
                try {
                    String sourceValue = map.getSource() instanceof UriType ? map.getSourceUriType().getValue() : map.getSourceReference().getReference();
                    mapping.setSourceValueSet(sourceValue);
                    String targetValue = map.getTarget() instanceof UriType ? map.getTargetUriType().getValue() : map.getTargetReference().getReference();
                    mapping.setTargetValueSet(targetValue);
                } catch (FHIRException fhirException) {
                    // an exception.
                    throw new RuntimeException(fhirException);
                }
                mapping.setSourceSystem(group.getSource());
                mapping.setSourceValue(element.getCode());
                mapping.setTargetSystem(group.getTarget());
                mapping.setTargetValue(target.getCode());
                if (target.getEquivalence() != null) {
                    mapping.setEquivalence(target.getEquivalence().toCode());
                }
                mappings.add(mapping);
            }
        }
    }
    return mappings.iterator();
}
Also used : ArrayList(java.util.ArrayList) TargetElementComponent(org.hl7.fhir.dstu3.model.ConceptMap.TargetElementComponent) Mapping(com.cerner.bunsen.spark.codes.Mapping) FHIRException(org.hl7.fhir.exceptions.FHIRException) ConceptMapGroupComponent(org.hl7.fhir.dstu3.model.ConceptMap.ConceptMapGroupComponent) SourceElementComponent(org.hl7.fhir.dstu3.model.ConceptMap.SourceElementComponent) UriType(org.hl7.fhir.dstu3.model.UriType)

Example 5 with Mapping

use of com.cerner.bunsen.spark.codes.Mapping in project bunsen by cerner.

the class ConceptMaps method broadcast.

@Override
public Broadcast<BroadcastableMappings> broadcast(Map<String, String> conceptMapUriToVersion) {
    List<ConceptMap> mapsList = getMaps().collectAsList().stream().map(row -> (ConceptMap) conceptMapRowConverter.rowToResource(row)).collect(Collectors.toList());
    Map<String, ConceptMap> mapsToLoad = mapsList.stream().filter(conceptMap -> conceptMap.getVersion().equals(conceptMapUriToVersion.get(conceptMap.getUrl()))).collect(Collectors.toMap(ConceptMap::getUrl, java.util.function.Function.identity()));
    // Expand the concept maps to load and sort them so dependencies are before
    // their dependents in the list.
    List<String> sortedMapsToLoad = sortMapsToLoad(conceptMapUriToVersion.keySet(), mapsToLoad);
    // Since this is used to map from one system to another, we use only targets
    // that don't introduce inaccurate meanings. (For instance, we can't map
    // general condition code to a more specific type, since that is not
    // representative of the source data.)
    Dataset<Mapping> mappings = getMappings(conceptMapUriToVersion).filter("equivalence in ('equivalent', 'equals', 'wider', 'subsumes')");
    // Group mappings by their concept map URI
    Map<String, List<Mapping>> groupedMappings = mappings.collectAsList().stream().collect(Collectors.groupingBy(Mapping::getConceptMapUri));
    Map<String, BroadcastableConceptMap> broadcastableMaps = new HashMap<>();
    for (String conceptMapUri : sortedMapsToLoad) {
        ConceptMap map = mapsToLoad.get(conceptMapUri);
        Set<String> children = getMapChildren(map);
        List<BroadcastableConceptMap> childMaps = children.stream().map(child -> broadcastableMaps.get(child)).collect(Collectors.toList());
        BroadcastableConceptMap broadcastableConceptMap = new BroadcastableConceptMap(conceptMapUri, groupedMappings.getOrDefault(conceptMapUri, Collections.emptyList()), childMaps);
        broadcastableMaps.put(conceptMapUri, broadcastableConceptMap);
    }
    JavaSparkContext ctx = new JavaSparkContext(getMaps().sparkSession().sparkContext());
    return ctx.broadcast(new BroadcastableMappings(broadcastableMaps));
}
Also used : SparkRowConverter(com.cerner.bunsen.spark.SparkRowConverter) Dataset(org.apache.spark.sql.Dataset) JavaSparkContext(org.apache.spark.api.java.JavaSparkContext) ConceptMapGroupComponent(org.hl7.fhir.dstu3.model.ConceptMap.ConceptMapGroupComponent) HashMap(java.util.HashMap) Deque(java.util.Deque) TargetElementComponent(org.hl7.fhir.dstu3.model.ConceptMap.TargetElementComponent) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) Mapping(com.cerner.bunsen.spark.codes.Mapping) BroadcastableConceptMap(com.cerner.bunsen.spark.codes.broadcast.BroadcastableConceptMap) UrlAndVersion(com.cerner.bunsen.spark.codes.UrlAndVersion) FhirVersionEnum(ca.uhn.fhir.context.FhirVersionEnum) Map(java.util.Map) org.apache.spark.sql.functions.lit(org.apache.spark.sql.functions.lit) ConceptMap(org.hl7.fhir.dstu3.model.ConceptMap) BroadcastableMappings(com.cerner.bunsen.spark.codes.broadcast.BroadcastableMappings) JavaRDD(org.apache.spark.api.java.JavaRDD) SparkSession(org.apache.spark.sql.SparkSession) Broadcast(org.apache.spark.broadcast.Broadcast) SourceElementComponent(org.hl7.fhir.dstu3.model.ConceptMap.SourceElementComponent) Iterator(java.util.Iterator) UriType(org.hl7.fhir.dstu3.model.UriType) Set(java.util.Set) Row(org.apache.spark.sql.Row) Collectors(java.util.stream.Collectors) List(java.util.List) Encoder(org.apache.spark.sql.Encoder) ConceptMapGroupUnmappedMode(org.hl7.fhir.dstu3.model.ConceptMap.ConceptMapGroupUnmappedMode) FhirContexts(com.cerner.bunsen.FhirContexts) FHIRException(org.hl7.fhir.exceptions.FHIRException) Function(org.apache.spark.api.java.function.Function) ArrayDeque(java.util.ArrayDeque) AbstractConceptMaps(com.cerner.bunsen.spark.codes.base.AbstractConceptMaps) Collections(java.util.Collections) BroadcastableConceptMap(com.cerner.bunsen.spark.codes.broadcast.BroadcastableConceptMap) HashMap(java.util.HashMap) Mapping(com.cerner.bunsen.spark.codes.Mapping) BroadcastableMappings(com.cerner.bunsen.spark.codes.broadcast.BroadcastableMappings) ArrayList(java.util.ArrayList) List(java.util.List) JavaSparkContext(org.apache.spark.api.java.JavaSparkContext) BroadcastableConceptMap(com.cerner.bunsen.spark.codes.broadcast.BroadcastableConceptMap) ConceptMap(org.hl7.fhir.dstu3.model.ConceptMap)

Aggregations

Mapping (com.cerner.bunsen.spark.codes.Mapping)7 ConceptMap (org.hl7.fhir.dstu3.model.ConceptMap)4 ConceptMapGroupComponent (org.hl7.fhir.dstu3.model.ConceptMap.ConceptMapGroupComponent)3 SourceElementComponent (org.hl7.fhir.dstu3.model.ConceptMap.SourceElementComponent)3 Test (org.junit.Test)3 FhirVersionEnum (ca.uhn.fhir.context.FhirVersionEnum)2 FhirContexts (com.cerner.bunsen.FhirContexts)2 SparkRowConverter (com.cerner.bunsen.spark.SparkRowConverter)2 UrlAndVersion (com.cerner.bunsen.spark.codes.UrlAndVersion)2 BroadcastableMappings (com.cerner.bunsen.spark.codes.broadcast.BroadcastableMappings)2 ArrayList (java.util.ArrayList)2 List (java.util.List)2 Map (java.util.Map)2 Set (java.util.Set)2 Collectors (java.util.stream.Collectors)2 JavaRDD (org.apache.spark.api.java.JavaRDD)2 JavaSparkContext (org.apache.spark.api.java.JavaSparkContext)2 Function (org.apache.spark.api.java.function.Function)2 Broadcast (org.apache.spark.broadcast.Broadcast)2 Dataset (org.apache.spark.sql.Dataset)2