Examples with DatasetImpl org.globalbioticinteractions.dataset.DatasetImpl used on opensource projects

Example 16 with DatasetImpl

use of org.globalbioticinteractions.dataset.DatasetImpl in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCATest method importRecordsFromArctosArchive.

@Test
public void importRecordsFromArctosArchive() throws StudyImporterException, URISyntaxException {
    URL resource = getClass().getResource("/org/globalbioticinteractions/dataset/arctos_mvz_bird_small.zip");
    DatasetImporterForDwCA studyImporterForDwCA = new DatasetImporterForDwCA(null, null);
    studyImporterForDwCA.setDataset(new DatasetImpl("some/namespace", resource.toURI(), inStream -> inStream));
    List<String> families = new ArrayList<>();
    AtomicBoolean someRecords = new AtomicBoolean(false);
    studyImporterForDwCA.setInteractionListener(new InteractionListener() {

        @Override
        public void on(Map<String, String> interaction) throws StudyImporterException {
            assertThat(interaction.get(REFERENCE_URL), startsWith("http://arctos.database.museum/guid/"));
            assertThat(interaction.get(SOURCE_OCCURRENCE_ID), anyOf(is("http://arctos.database.museum/guid/MVZ:Bird:180448?seid=587053"), is("http://arctos.database.museum/guid/MVZ:Bird:183644?seid=158590"), is("http://arctos.database.museum/guid/MVZ:Bird:58090?seid=657121")));
            if (interaction.containsKey(DatasetImporterForTSV.TARGET_OCCURRENCE_ID)) {
                assertThat(interaction.get(DatasetImporterForTSV.TARGET_OCCURRENCE_ID), anyOf(is("http://arctos.database.museum/guid/MVZ:Herp:241200"), is("http://arctos.database.museum/guid/MVZ:Bird:183643"), is("http://arctos.database.museum/guid/MVZ:Bird:58093")));
            }
            assertThat(interaction.get(SOURCE_TAXON_FAMILY), anyOf(is("Accipitridae"), is("Strigidae")));
            assertThat(interaction.get(DatasetImporterForTSV.RESOURCE_TYPES), is("http://rs.tdwg.org/dwc/terms/associatedOccurrences | http://rs.tdwg.org/dwc/terms/Occurrence"));
            someRecords.set(true);
        }
    });
    studyImporterForDwCA.importStudy();
    assertThat(someRecords.get(), is(true));
}

Example 17 with DatasetImpl

use of org.globalbioticinteractions.dataset.DatasetImpl in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCATest method importRecordsFromMCZ.

@Test
public void importRecordsFromMCZ() throws StudyImporterException, URISyntaxException {
    StringBuilder actualMessage = new StringBuilder();
    URL resource = getClass().getResource("/org/globalbioticinteractions/dataset/mcz/meta.xml");
    URI archiveRoot = new File(resource.toURI()).getParentFile().toURI();
    AtomicInteger recordCounter = new AtomicInteger(0);
    DatasetImporterForDwCA studyImporterForDwCA = new DatasetImporterForDwCA(null, null);
    studyImporterForDwCA.setLogger(new NullImportLogger() {

        @Override
        public void severe(LogContext ctx, String message) {
            actualMessage.append(message);
        }
    });
    studyImporterForDwCA.setDataset(new DatasetImpl("some/namespace", archiveRoot, inStream -> inStream));
    studyImporterForDwCA.setInteractionListener(new InteractionListener() {

        @Override
        public void on(Map<String, String> interaction) throws StudyImporterException {
            for (String expectedProperty : new String[] {}) {
                assertThat("no [" + expectedProperty + "] found in " + interaction, interaction.containsKey(expectedProperty), is(true));
                assertThat("no value of [" + expectedProperty + "] found in " + interaction, interaction.get(expectedProperty), is(notNullValue()));
            }
            assertThat(interaction.get(DatasetImporterForTSV.RESOURCE_TYPES), is("http://rs.tdwg.org/dwc/terms/ResourceRelationship | http://rs.tdwg.org/dwc/terms/Occurrence"));
            recordCounter.incrementAndGet();
        }
    });
    studyImporterForDwCA.importStudy();
    assertThat(recordCounter.get(), is(0));
    assertThat(actualMessage.toString(), startsWith("[failed to handle dwc record]"));
}

Example 18 with DatasetImpl

use of org.globalbioticinteractions.dataset.DatasetImpl in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCATest method importRecordsFromResourceRelationshipArchiveRemarksOnly.

@Test
public void importRecordsFromResourceRelationshipArchiveRemarksOnly() throws StudyImporterException, URISyntaxException {
    URL resource = getClass().getResource("fmnh-rr-8278596f-4d3f-4f82-8cd1-b5070fe1bc7c.zip");
    AtomicInteger recordCounter = new AtomicInteger(0);
    DatasetImporterForDwCA studyImporterForDwCA = new DatasetImporterForDwCA(null, null);
    studyImporterForDwCA.setDataset(new DatasetImpl("some/namespace", resource.toURI(), inStream -> inStream));
    studyImporterForDwCA.setInteractionListener(interaction -> {
        assertThat(interaction.get(TARGET_TAXON_NAME), is("Glaucomys volans"));
        assertThat(interaction.get(TARGET_OCCURRENCE_ID), is(nullValue()));
        assertThat(interaction.get(SOURCE_TAXON_NAME), is("Orchopeas fulleri Traub, 1950"));
        assertThat(interaction.get(SOURCE_OCCURRENCE_ID), is("8278596f-4d3f-4f82-8cd1-b5070fe1bc7c"));
        recordCounter.incrementAndGet();
        assertThat(interaction.get(DatasetImporterForTSV.RESOURCE_TYPES), is("http://rs.tdwg.org/dwc/terms/ResourceRelationship | http://rs.tdwg.org/dwc/terms/Occurrence"));
    });
    studyImporterForDwCA.importStudy();
    assertThat(recordCounter.get(), greaterThan(0));
}

Example 19 with DatasetImpl

use of org.globalbioticinteractions.dataset.DatasetImpl in project eol-globi-data by jhpoelen.

the class DatasetImporterForDwCATest method importRecords.

@Test
public void importRecords() throws StudyImporterException, URISyntaxException, IOException {
    URL resource = getClass().getResource("/org/globalbioticinteractions/dataset/dwca.zip");
    DatasetImporterForDwCA studyImporterForDwCA = new DatasetImporterForDwCA(null, null);
    DatasetImpl dataset = new DatasetImpl("some/namespace", resource.toURI(), inStream -> inStream);
    dataset.setConfig(new ObjectMapper().readTree("{ \"citation\": \"some citation\" }"));
    studyImporterForDwCA.setDataset(dataset);
    AtomicBoolean someRecords = new AtomicBoolean(false);
    Set<String> resourceTypes = new TreeSet<>();
    studyImporterForDwCA.setInteractionListener(new InteractionListener() {

        @Override
        public void on(Map<String, String> interaction) throws StudyImporterException {
            String associatedTaxa = interaction.get("http://rs.tdwg.org/dwc/terms/associatedTaxa");
            String dynamicProperties = interaction.get("http://rs.tdwg.org/dwc/terms/dynamicProperties");
            assertThat(StringUtils.isNotBlank(associatedTaxa) || StringUtils.isNotBlank(dynamicProperties), is(true));
            assertThat(interaction.get(SOURCE_TAXON_NAME), is(not(nullValue())));
            assertThat(interaction.get(TaxonUtil.TARGET_TAXON_NAME), is(not(nullValue())));
            assertThat(interaction.get(INTERACTION_TYPE_NAME), is(not(nullValue())));
            assertThat(interaction.get(DatasetImporterForTSV.DATASET_CITATION), containsString("some citation"));
            assertThat(interaction.get(DatasetImporterForTSV.DATASET_CITATION), containsString("Accessed at"));
            assertThat(interaction.get(DatasetImporterForTSV.DATASET_CITATION), containsString("dataset/dwca.zip"));
            assertThat(interaction.get(REFERENCE_ID), is(not(nullValue())));
            assertThat(interaction.get(DatasetImporterForTSV.REFERENCE_CITATION), is(not(nullValue())));
            assertThat(interaction.get(REFERENCE_URL), is(not(nullValue())));
            resourceTypes.addAll(Arrays.asList(splitByPipes(interaction.get(RESOURCE_TYPES))));
            someRecords.set(true);
        }
    });
    studyImporterForDwCA.importStudy();
    assertThat(someRecords.get(), is(true));
    assertThat(resourceTypes, containsInAnyOrder("http://rs.tdwg.org/dwc/terms/dynamicProperties", "http://rs.tdwg.org/dwc/terms/Occurrence", "http://rs.tdwg.org/dwc/terms/associatedTaxa"));
}

Example 20 with DatasetImpl

use of org.globalbioticinteractions.dataset.DatasetImpl in project eol-globi-data by jhpoelen.

the class DatasetImporterForRegistryTest method filteredDatasets.

@Test
public void filteredDatasets() throws StudyImporterException {
    DatasetImporterForRegistry importer = new DatasetImporterForRegistry(null, null, new DatasetRegistry() {

        @Override
        public Collection<String> findNamespaces() throws DatasetRegistryException {
            return Collections.singletonList("some/namespace");
        }

        @Override
        public Dataset datasetFor(String namespace) throws DatasetRegistryException {
            DatasetImpl dataset = new DatasetImpl("some/namespace", URI.create("some:uri"), in -> in) {

                @Override
                public InputStream retrieve(URI resource) throws IOException {
                    if (!StringUtils.endsWith(resource.toString(), "globi.json")) {
                        throw new IOException();
                    }
                    return IOUtils.toInputStream("{\"some\":\"thing\"}", StandardCharsets.UTF_8);
                }
            };
            return dataset;
        }
    });
    importer.setDatasetFilter(x -> false);
    importer.importStudy();
}