Search in sources :

Example 1 with Term

use of org.gbif.dwc.terms.Term in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCA method termsToMap.

private static void termsToMap(Record record, Map<String, String> props) {
    for (Term term : record.terms()) {
        String value = record.value(term);
        if (StringUtils.isNotBlank(value)) {
            props.put(term.qualifiedName(), value);
        }
        appendResourceType(props, record.rowType());
    }
}
Also used : DwcTerm(org.gbif.dwc.terms.DwcTerm) Term(org.gbif.dwc.terms.Term) DcTerm(org.gbif.dwc.terms.DcTerm)

Example 2 with Term

use of org.gbif.dwc.terms.Term in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCA method handleRecord.

private void handleRecord(InteractionListener interactionListener, Record rec) throws StudyImporterException {
    List<Map<String, String>> interactionCandidates = new ArrayList<>();
    appendInteractionCandidatesIfAvailable(rec, interactionCandidates, DwcTerm.associatedTaxa, "");
    appendInteractionCandidatesIfAvailable(rec, interactionCandidates, DwcTerm.habitat, null);
    appendAssociatedOccurrencesIfAvailable(rec, interactionCandidates);
    // see https://github.com/globalbioticinteractions/globalbioticinteractions/issues/755#issuecomment-1029509362
    appendAssociatedSequencesIfAvailable(rec, interactionCandidates);
    String occurrenceRemarks = rec.value(DwcTerm.occurrenceRemarks);
    try {
        addCandidatesFromRemarks(interactionCandidates, occurrenceRemarks);
    } catch (IOException e) {
        if (getLogger() != null) {
            Map<String, String> interactionProperties = new HashMap<>();
            mapCoreProperties(rec, interactionProperties);
            getLogger().warn(LogUtil.contextFor(interactionProperties), e.getMessage());
        }
    }
    String dynamicProperties = rec.value(DwcTerm.dynamicProperties);
    if (StringUtils.isNotBlank(dynamicProperties)) {
        Map<String, String> parsedDynamicProperties = parseDynamicPropertiesForInteractionsOnly(dynamicProperties);
        if (!parsedDynamicProperties.isEmpty()) {
            interactionCandidates.add(parsedDynamicProperties);
        }
    }
    Map<String, String> interaction = new HashMap<>(rec.terms().size());
    for (Term term : rec.terms()) {
        interaction.put(term.qualifiedName(), rec.value(term));
    }
    if (interactionCandidates.isEmpty() && isDependency()) {
        // If no candidates are found,
        // add empty candidate to allow interaction listeners to do indexing/enriching
        interactionCandidates.add(new TreeMap<>());
    }
    for (Map<String, String> interactionProperties : interactionCandidates) {
        interactionProperties.putAll(interaction);
        interactionProperties.put(DWC_COREID, rec.id());
        mapIfAvailable(rec, interactionProperties, BASIS_OF_RECORD_NAME, DwcTerm.basisOfRecord);
        mapCoreProperties(rec, interactionProperties);
        appendResourceType(interactionProperties, rec.rowType());
        interactionListener.on(interactionProperties);
    }
}
Also used : HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) IOException(java.io.IOException) DwcTerm(org.gbif.dwc.terms.DwcTerm) Term(org.gbif.dwc.terms.Term) DcTerm(org.gbif.dwc.terms.DcTerm) Map(java.util.Map) HashMap(java.util.HashMap) BTreeMap(org.mapdb.BTreeMap) TreeMap(java.util.TreeMap)

Example 3 with Term

use of org.gbif.dwc.terms.Term in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCA method importInteractionsFromResourceRelationships.

private static void importInteractionsFromResourceRelationships(InteractionListener interactionListener, ArchiveFile resourceExtension, Map<String, Map<String, Map<String, String>>> termTypeIdPropMap, List<DwcTerm> termTypes) {
    for (Record record : resourceExtension) {
        Map<String, String> props = new TreeMap<>();
        appendResourceType(props, resourceExtension.getRowType());
        String sourceId = record.value(DwcTerm.resourceID);
        String relationship = record.value(DwcTerm.relationshipOfResource);
        Optional<Term> relationshipOfResourceIDTerm = record.terms().stream().filter(x -> StringUtils.equals(x.simpleName(), "relationshipOfResourceID")).findFirst();
        String relationshipTypeIdValue = relationshipOfResourceIDTerm.map(record::value).orElse(null);
        String targetId = record.value(DwcTerm.relatedResourceID);
        if (StringUtils.isNotBlank(sourceId)) {
            appendVerbatimResourceRelationsValues(record, props);
            String relationshipAccordingTo = record.value(DwcTerm.relationshipAccordingTo);
            if (StringUtils.isNotBlank(relationshipAccordingTo)) {
                props.putIfAbsent(REFERENCE_CITATION, relationshipAccordingTo);
            }
            putIfAbsentAndNotBlank(props, INTERACTION_TYPE_NAME, relationship);
            putIfAbsentAndNotBlank(props, INTERACTION_TYPE_ID, relationshipTypeIdValue);
            putIfAbsentAndNotBlank(props, DatasetImporterForMetaTable.EVENT_DATE, record.value(DwcTerm.relationshipEstablishedDate));
            for (DwcTerm termType : termTypes) {
                String key = termType.qualifiedName();
                if (StringUtils.isNoneBlank(key) && termTypeIdPropMap.containsKey(key)) {
                    Map<String, Map<String, String>> propMap = termTypeIdPropMap.get(termType.qualifiedName());
                    populatePropertiesAssociatedWithId(props, sourceId, true, propMap.get(sourceId), labelPairFor(termType));
                    extractNameFromRelationshipRemarks(record).ifPresent(name -> props.put(TARGET_TAXON_NAME, name));
                    populatePropertiesAssociatedWithId(props, targetId, false, propMap.get(targetId), labelPairFor(termType));
                }
            }
            try {
                interactionListener.on(props);
            } catch (StudyImporterException e) {
            // 
            }
        }
    }
}
Also used : TaxonUtil(org.eol.globi.service.TaxonUtil) CacheUtil(org.globalbioticinteractions.cache.CacheUtil) Arrays(java.util.Arrays) INTERACTION_TYPE_NAME(org.eol.globi.data.DatasetImporterForTSV.INTERACTION_TYPE_NAME) URL(java.net.URL) URISyntaxException(java.net.URISyntaxException) INTERACTION_TYPE_ID(org.eol.globi.data.DatasetImporterForTSV.INTERACTION_TYPE_ID) InteractionListener(org.eol.globi.process.InteractionListener) ExternalIdUtil(org.eol.globi.util.ExternalIdUtil) SOURCE_TAXON_SUBGENUS(org.eol.globi.service.TaxonUtil.SOURCE_TAXON_SUBGENUS) StringUtils(org.apache.commons.lang3.StringUtils) SOURCE_TAXON_ORDER(org.eol.globi.service.TaxonUtil.SOURCE_TAXON_ORDER) InteractType(org.eol.globi.domain.InteractType) REFERENCE_URL(org.eol.globi.data.DatasetImporterForTSV.REFERENCE_URL) Matcher(java.util.regex.Matcher) Pair(org.apache.commons.lang3.tuple.Pair) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DECIMAL_LONGITUDE(org.eol.globi.data.DatasetImporterForTSV.DECIMAL_LONGITUDE) TARGET_TAXON_ID(org.eol.globi.service.TaxonUtil.TARGET_TAXON_ID) Map(java.util.Map) SOURCE_TAXON_ID(org.eol.globi.service.TaxonUtil.SOURCE_TAXON_ID) SOURCE_LIFE_STAGE_NAME(org.eol.globi.data.DatasetImporterForTSV.SOURCE_LIFE_STAGE_NAME) JsonNode(com.fasterxml.jackson.databind.JsonNode) URI(java.net.URI) TARGET_LIFE_STAGE_NAME(org.eol.globi.data.DatasetImporterForTSV.TARGET_LIFE_STAGE_NAME) MapDBUtil(org.globalbioticinteractions.util.MapDBUtil) Path(java.nio.file.Path) TARGET_OCCURRENCE_ID(org.eol.globi.data.DatasetImporterForTSV.TARGET_OCCURRENCE_ID) SOURCE_TAXON_NAME(org.eol.globi.service.TaxonUtil.SOURCE_TAXON_NAME) DwcTerm(org.gbif.dwc.terms.DwcTerm) TARGET_FIELD_NUMBER(org.eol.globi.data.DatasetImporterForTSV.TARGET_FIELD_NUMBER) ArchiveFile(org.gbif.dwc.ArchiveFile) LOCALITY_NAME(org.eol.globi.data.DatasetImporterForTSV.LOCALITY_NAME) REFERENCE_ID(org.eol.globi.data.DatasetImporterForTSV.REFERENCE_ID) Record(org.gbif.dwc.record.Record) Set(java.util.Set) StandardCharsets(java.nio.charset.StandardCharsets) IOUtils(org.apache.commons.io.IOUtils) TARGET_TAXON_PATH_NAMES(org.eol.globi.service.TaxonUtil.TARGET_TAXON_PATH_NAMES) List(java.util.List) SOURCE_TAXON_GENUS(org.eol.globi.service.TaxonUtil.SOURCE_TAXON_GENUS) SOURCE_TAXON_SPECIFIC_EPITHET(org.eol.globi.service.TaxonUtil.SOURCE_TAXON_SPECIFIC_EPITHET) Optional(java.util.Optional) Pattern(java.util.regex.Pattern) TARGET_BODY_PART_NAME(org.eol.globi.data.DatasetImporterForTSV.TARGET_BODY_PART_NAME) DATASET_CITATION(org.eol.globi.data.DatasetImporterForTSV.DATASET_CITATION) TARGET_TAXON_GENUS(org.eol.globi.service.TaxonUtil.TARGET_TAXON_GENUS) CitationUtil(org.globalbioticinteractions.dataset.CitationUtil) TARGET_CATALOG_NUMBER(org.eol.globi.data.DatasetImporterForTSV.TARGET_CATALOG_NUMBER) SOURCE_TAXON_FAMILY(org.eol.globi.service.TaxonUtil.SOURCE_TAXON_FAMILY) DatasetConstant(org.globalbioticinteractions.dataset.DatasetConstant) HashMap(java.util.HashMap) SOURCE_TAXON_CLASS(org.eol.globi.service.TaxonUtil.SOURCE_TAXON_CLASS) TARGET_TAXON_CLASS(org.eol.globi.service.TaxonUtil.TARGET_TAXON_CLASS) ArrayList(java.util.ArrayList) SOURCE_SEX_NAME(org.eol.globi.data.DatasetImporterForTSV.SOURCE_SEX_NAME) TARGET_TAXON_KINGDOM(org.eol.globi.service.TaxonUtil.TARGET_TAXON_KINGDOM) TARGET_TAXON_SPECIFIC_EPITHET(org.eol.globi.service.TaxonUtil.TARGET_TAXON_SPECIFIC_EPITHET) InteractionListenerClosable(org.eol.globi.process.InteractionListenerClosable) SOURCE_TAXON_KINGDOM(org.eol.globi.service.TaxonUtil.SOURCE_TAXON_KINGDOM) RESOURCE_TYPES(org.eol.globi.data.DatasetImporterForTSV.RESOURCE_TYPES) SOURCE_OCCURRENCE_ID(org.eol.globi.data.DatasetImporterForTSV.SOURCE_OCCURRENCE_ID) Term(org.gbif.dwc.terms.Term) MapUtils(org.apache.commons.collections4.MapUtils) BASIS_OF_RECORD_NAME(org.eol.globi.data.DatasetImporterForTSV.BASIS_OF_RECORD_NAME) ClosableIterator(org.gbif.utils.file.ClosableIterator) TARGET_TAXON_ORDER(org.eol.globi.service.TaxonUtil.TARGET_TAXON_ORDER) Iterator(java.util.Iterator) MalformedURLException(java.net.MalformedURLException) Files(java.nio.file.Files) ObjectMapper(com.fasterxml.jackson.databind.ObjectMapper) FileUtils(org.apache.commons.io.FileUtils) Archive(org.gbif.dwc.Archive) DwCAUtil(org.globalbioticinteractions.dataset.DwCAUtil) IOException(java.io.IOException) TARGET_TAXON_NAME(org.eol.globi.service.TaxonUtil.TARGET_TAXON_NAME) File(java.io.File) LOCALITY_ID(org.eol.globi.data.DatasetImporterForTSV.LOCALITY_ID) SOURCE_TAXON_PHYLUM(org.eol.globi.service.TaxonUtil.SOURCE_TAXON_PHYLUM) TARGET_TAXON_FAMILY(org.eol.globi.service.TaxonUtil.TARGET_TAXON_FAMILY) TARGET_SEX_NAME(org.eol.globi.data.DatasetImporterForTSV.TARGET_SEX_NAME) BTreeMap(org.mapdb.BTreeMap) TreeMap(java.util.TreeMap) DECIMAL_LATITUDE(org.eol.globi.data.DatasetImporterForTSV.DECIMAL_LATITUDE) DcTerm(org.gbif.dwc.terms.DcTerm) TARGET_TAXON_PATH(org.eol.globi.service.TaxonUtil.TARGET_TAXON_PATH) TARGET_BODY_PART_ID(org.eol.globi.data.DatasetImporterForTSV.TARGET_BODY_PART_ID) TARGET_TAXON_PHYLUM(org.eol.globi.service.TaxonUtil.TARGET_TAXON_PHYLUM) DB(org.mapdb.DB) Collections(java.util.Collections) InputStream(java.io.InputStream) REFERENCE_CITATION(org.eol.globi.data.DatasetImporterForTSV.REFERENCE_CITATION) Record(org.gbif.dwc.record.Record) DwcTerm(org.gbif.dwc.terms.DwcTerm) Term(org.gbif.dwc.terms.Term) DcTerm(org.gbif.dwc.terms.DcTerm) BTreeMap(org.mapdb.BTreeMap) TreeMap(java.util.TreeMap) Map(java.util.Map) HashMap(java.util.HashMap) BTreeMap(org.mapdb.BTreeMap) TreeMap(java.util.TreeMap) DwcTerm(org.gbif.dwc.terms.DwcTerm)

Aggregations

DcTerm (org.gbif.dwc.terms.DcTerm)3 DwcTerm (org.gbif.dwc.terms.DwcTerm)3 Term (org.gbif.dwc.terms.Term)3 IOException (java.io.IOException)2 ArrayList (java.util.ArrayList)2 HashMap (java.util.HashMap)2 Map (java.util.Map)2 TreeMap (java.util.TreeMap)2 BTreeMap (org.mapdb.BTreeMap)2 JsonNode (com.fasterxml.jackson.databind.JsonNode)1 ObjectMapper (com.fasterxml.jackson.databind.ObjectMapper)1 File (java.io.File)1 InputStream (java.io.InputStream)1 MalformedURLException (java.net.MalformedURLException)1 URI (java.net.URI)1 URISyntaxException (java.net.URISyntaxException)1 URL (java.net.URL)1 StandardCharsets (java.nio.charset.StandardCharsets)1 Files (java.nio.file.Files)1 Path (java.nio.file.Path)1