Examples with Record org.gbif.dwc.record.Record used on opensource projects

Example 1 with Record

use of org.gbif.dwc.record.Record in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCATest method importRecordsFromMCZ.

@Test
public void importRecordsFromMCZ() throws StudyImporterException, URISyntaxException {
    StringBuilder actualMessage = new StringBuilder();
    URL resource = getClass().getResource("/org/globalbioticinteractions/dataset/mcz/meta.xml");
    URI archiveRoot = new File(resource.toURI()).getParentFile().toURI();
    AtomicInteger recordCounter = new AtomicInteger(0);
    DatasetImporterForDwCA studyImporterForDwCA = new DatasetImporterForDwCA(null, null);
    studyImporterForDwCA.setLogger(new NullImportLogger() {

        @Override
        public void severe(LogContext ctx, String message) {
            actualMessage.append(message);
        }
    });
    studyImporterForDwCA.setDataset(new DatasetImpl("some/namespace", archiveRoot, inStream -> inStream));
    studyImporterForDwCA.setInteractionListener(new InteractionListener() {

        @Override
        public void on(Map<String, String> interaction) throws StudyImporterException {
            for (String expectedProperty : new String[] {}) {
                assertThat("no [" + expectedProperty + "] found in " + interaction, interaction.containsKey(expectedProperty), is(true));
                assertThat("no value of [" + expectedProperty + "] found in " + interaction, interaction.get(expectedProperty), is(notNullValue()));
            }
            assertThat(interaction.get(DatasetImporterForTSV.RESOURCE_TYPES), is("http://rs.tdwg.org/dwc/terms/ResourceRelationship | http://rs.tdwg.org/dwc/terms/Occurrence"));
            recordCounter.incrementAndGet();
        }
    });
    studyImporterForDwCA.importStudy();
    assertThat(recordCounter.get(), is(0));
    assertThat(actualMessage.toString(), startsWith("[failed to handle dwc record]"));
}

Example 2 with Record

use of org.gbif.dwc.record.Record in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCA method createReferenceEnricher.

private static InteractionListenerClosable createReferenceEnricher(Archive archive, final InteractionListener interactionListener) {
    return new InteractionListenerClosable() {

        private BTreeMap<String, Map<String, String>> referenceMap = null;

        @Override
        public void close() {
            if (referenceMap != null) {
                referenceMap.close();
                referenceMap = null;
            }
        }

        private void initIfNeeded() {
            if (referenceMap == null) {
                referenceMap = MapDBUtil.createBigMap();
                ArchiveFile extension = findResourceExtension(archive, EXTENSION_REFERENCE);
                if (extension != null) {
                    for (Record record : extension) {
                        Map<String, String> props = new TreeMap<>();
                        termsToMap(record, props);
                        props.put(REFERENCE_CITATION, CitationUtil.citationFor(props));
                        appendResourceType(props, extension.getRowType());
                        referenceMap.put(record.id(), props);
                    }
                }
            }
        }

        @Override
        public void on(Map<String, String> interaction) throws StudyImporterException {
            initIfNeeded();
            String s = interaction.get(DWC_COREID);
            Map<String, String> enrichedLink = contains(referenceMap, s) ? new TreeMap<String, String>(interaction) {

                {
                    putAll(referenceMap.get(s));
                }
            } : interaction;
            interactionListener.on(enrichedLink);
        }
    };
}

Example 3 with Record

use of org.gbif.dwc.record.Record in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCA method resolveLocalResourceIds.

private static void resolveLocalResourceIds(Archive archive, Map<String, Map<String, Map<String, String>>> termIdPropMap, Set<String> referencedSourceIds, Set<String> referencedTargetIds, List<DwcTerm> termTypes) {
    List<ArchiveFile> archiveFiles = new ArrayList<>();
    archiveFiles.add(archive.getCore());
    ArchiveFile taxon = findResourceExtension(archive, EXTENSION_TAXON);
    if (taxon != null) {
        archiveFiles.add(taxon);
    }
    for (ArchiveFile archiveFile : archiveFiles) {
        for (Record record : archiveFile) {
            for (DwcTerm termType : termTypes) {
                attemptLinkUsingTerm(termIdPropMap, referencedSourceIds, referencedTargetIds, record, termType);
            }
        }
    }
}

Example 4 with Record

use of org.gbif.dwc.record.Record in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCA method importDescriptionExtension.

private static void importDescriptionExtension(InteractionListener interactionListener, ImportLogger logger, ArchiveFile extension, ArchiveFile core, BTreeMap<String, Map<String, String>> associationsMap) {
    for (Record record : extension) {
        Map<String, String> props = new TreeMap<>();
        termsToMap(record, props);
        associationsMap.put(record.id(), props);
    }
    for (Record coreRecord : core) {
        String id = coreRecord.id();
        if (contains(associationsMap, id)) {
            try {
                Map<String, String> targetProperties = associationsMap.get(id);
                String referenceCitation = targetProperties.get("http://purl.org/dc/terms/source");
                String descriptionType = targetProperties.get("http://purl.org/dc/terms/type");
                if (isUnsupportedDescriptionType(descriptionType)) {
                    if (logger != null) {
                        logger.info(null, "ignoring unsupported taxon description of type [" + descriptionType + "]");
                    }
                } else {
                    String interactionTypeNameDefault = isEcologyDescription(descriptionType) ? null : "";
                    List<Map<String, String>> maps = AssociatedTaxaUtil.parseAssociatedTaxa(targetProperties.get("http://purl.org/dc/terms/description"), interactionTypeNameDefault);
                    for (Map<String, String> map : maps) {
                        TreeMap<String, String> interaction = new TreeMap<>(map);
                        interaction.put(DWC_COREID, id);
                        mapCoreProperties(coreRecord, interaction);
                        if (StringUtils.isNotBlank(referenceCitation)) {
                            interaction.put(REFERENCE_CITATION, referenceCitation);
                            String urlString = ExternalIdUtil.urlForExternalId(referenceCitation);
                            if (ExternalIdUtil.isSupported(urlString)) {
                                interaction.put(REFERENCE_URL, urlString);
                            }
                        }
                        interactionListener.on(interaction);
                    }
                }
            } catch (StudyImporterException e) {
            // 
            }
        }
    }
}

Example 5 with Record

use of org.gbif.dwc.record.Record in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCATest method nonInteractionRecordMessage.

@Test
public void nonInteractionRecordMessage() throws StudyImporterException, URISyntaxException {
    List<String> msgs = new ArrayList<>();
    URL resource = getClass().getResource("/org/globalbioticinteractions/dataset/mcz-adjusted/meta.xml");
    URI archiveRoot = new File(resource.toURI()).getParentFile().toURI();
    AtomicInteger recordCounter = new AtomicInteger(0);
    DatasetImporterForDwCA studyImporterForDwCA = new DatasetImporterForDwCA(null, null);
    studyImporterForDwCA.setLogger(new NullImportLogger() {

        @Override
        public void info(LogContext ctx, String message) {
            msgs.add(message);
        }
    });
    studyImporterForDwCA.setDataset(new DatasetImpl("some/namespace", archiveRoot, inStream -> inStream));
    studyImporterForDwCA.setInteractionListener(new InteractionListener() {

        @Override
        public void on(Map<String, String> interaction) throws StudyImporterException {
            assertThat(interaction.get(DatasetImporterForTSV.RESOURCE_TYPES), is("http://rs.tdwg.org/dwc/terms/ResourceRelationship | http://rs.tdwg.org/dwc/terms/Occurrence"));
            recordCounter.incrementAndGet();
        }
    });
    studyImporterForDwCA.importStudy();
    assertThat(recordCounter.get(), is(0));
    String joinedMsgs = StringUtils.join(msgs, "\n");
    assertThat(joinedMsgs, containsString("]: indexing interaction records"));
    assertThat(joinedMsgs, containsString("]: scanned [1] record(s)"));
}