Examples with Dataset org.globalbioticinteractions.dataset.Dataset used on opensource projects

Example 36 with Dataset

use of org.globalbioticinteractions.dataset.Dataset in project eol-globi-data by jhpoelen.

the class NodeFactoryFactoryTransactingOnDatasetNeo4j2 method create.

@Override
public NodeFactory create(GraphDatabaseService service) {
    GraphDatabaseService graphService = graphServiceFactory.getGraphService();
    try (Transaction tx = graphService.beginTx()) {
        NodeFactory nodeFactory = new NodeFactoryNeo4j2(graphService) {

            final AtomicReference<Transaction> tx = new AtomicReference<>();

            final AtomicBoolean closing = new AtomicBoolean(false);

            final AtomicLong counter = new AtomicLong(0);

            @Override
            public Dataset getOrCreateDataset(Dataset dataset) {
                if (closing.get()) {
                    throw new IllegalStateException("cannot create a dataset on closing node factory");
                } else {
                    startBatchTransactionIfNeeded();
                }
                return super.getOrCreateDataset(dataset);
            }

            void startBatchTransactionIfNeeded() {
                tx.getAndUpdate(transaction -> {
                    if (counter.getAndIncrement() % TRANSACTION_BATCH_SIZE_DEFAULT == 0) {
                        if (transaction != null) {
                            transaction.success();
                            transaction.close();
                            transaction = null;
                        }
                    }
                    return transaction == null ? beginTx() : transaction;
                });
            }

            private Transaction beginTx() {
                return graphServiceFactory.getGraphService().beginTx();
            }

            @Override
            public SpecimenNode createSpecimen(Study study, Taxon taxon, RelTypes... types) throws NodeFactoryException {
                startBatchTransactionIfNeeded();
                return super.createSpecimen(study, taxon, types);
            }

            @Override
            public void close() {
                tx.getAndUpdate(tx -> {
                    closing.set(true);
                    if (tx != null) {
                        tx.success();
                        tx.close();
                    }
                    return null;
                });
            }
        };
        tx.success();
        return nodeFactory;
    }
}

Example 37 with Dataset

use of org.globalbioticinteractions.dataset.Dataset in project eol-globi-data by jhpoelen.

the class CmdGenerateReport method generateCollectionReport.

private void generateCollectionReport(DB reportCache) {
    final Set<Long> distinctTaxonIds = makeOrRemake(reportCache, "distinctTaxonIds");
    final Set<Long> distinctTaxonIdsNoMatch = makeOrRemake(reportCache, "distinctTaxonIdsNoMatch");
    final Counter counter = new Counter();
    final Counter studyCounter = new Counter();
    final Set<String> distinctSources = makeOrRemakeString(reportCache, "distinctSources");
    final Set<String> distinctDatasets = makeOrRemakeString(reportCache, "distinctDatasets");
    NodeUtil.findStudies(getGraphDb(), studyNode -> {
        countInteractionsAndTaxa(distinctTaxonIds, counter, distinctTaxonIdsNoMatch, studyNode);
        studyCounter.count();
        final Dataset originatingDataset = new StudyNode(studyNode).getOriginatingDataset();
        if (originatingDataset != null) {
            final String namespace = originatingDataset.getNamespace();
            distinctSources.add(namespace);
            distinctDatasets.add(namespace);
        }
    });
    final Node node = getGraphDb().createNode();
    node.setProperty(PropertyAndValueDictionary.COLLECTION, GLOBI_COLLECTION_NAME);
    node.setProperty(PropertyAndValueDictionary.NUMBER_OF_INTERACTIONS, counter.getCount() / 2);
    node.setProperty(PropertyAndValueDictionary.NUMBER_OF_DISTINCT_TAXA, distinctTaxonIds.size());
    node.setProperty(PropertyAndValueDictionary.NUMBER_OF_DISTINCT_TAXA_NO_MATCH, distinctTaxonIdsNoMatch.size());
    node.setProperty(PropertyAndValueDictionary.NUMBER_OF_STUDIES, studyCounter.getCount());
    node.setProperty(PropertyAndValueDictionary.NUMBER_OF_SOURCES, distinctSources.size());
    node.setProperty(PropertyAndValueDictionary.NUMBER_OF_DATASETS, distinctDatasets.size());
    getGraphDb().index().forNodes("reports").add(node, PropertyAndValueDictionary.COLLECTION, GLOBI_COLLECTION_NAME);
}

Example 38 with Dataset

use of org.globalbioticinteractions.dataset.Dataset in project eol-globi-data by jhpoelen.

the class DatasetImporterForMetaTableIT method importNHMStatic.

@Test
public void importNHMStatic() throws IOException, StudyImporterException {
    final List<Map<String, String>> links = new ArrayList<Map<String, String>>();
    final InteractionListener interactionListener = links::add;
    final DatasetImporterForMetaTable.TableParserFactory tableFactory = (config, dataset) -> {
        String firstFewLines = "\"InteractionID\",\"InteractionURL\",\"Species1UUID\",\"Species1Name\",\"Species1LifeCycleStage\",\"Species1OrganismPart\",\"Species1Status\",\"InteractionType\",\"InteractionOntologyURL\",\"Species2UUID\",\"Species2Name\",\"Species2LifeCycleStage\",\"Species2OrganismPart\",\"Species2Status\",\"LocationUUID\",\"LocationName\",\"LocationCountryName\",\"ISO2\",\"Importance\",\"InteractionRecordType\",\"Reference\",\"ReferenceDOI\",\"Reference Page\",\"Notes\"\n" + "\"4bee827f-c9f5-4c0e-9db3-e40a6e4d8008\",\"http://phthiraptera.info/node/94209\",\"c8faa033-237b-40b9-9b76-d9e7fcff9238\",\"Menacanthus alaudae\",\"\",\"\",\"\",\"ectoparasite of\",\"http://purl.obolibrary.org/obo/RO_0002632\",\"e275d77c-e993-4de0-981f-b3f39fd4da9b\",\"Acanthis flavirostris\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"310\",\"[REF: Palma, Price & Hellenthal, 1998:310]\"\n" + "\"80e66e7c-75db-467f-9a89-a11f94d58eb3\",\"http://phthiraptera.info/node/94210\",\"fe5b2e50-b414-41d9-840d-189e732b2ea5\",\"Ricinus fringillae flammeae\",\"\",\"\",\"\",\"ectoparasite of\",\"http://purl.obolibrary.org/obo/RO_0002632\",\"f26a1199-c0bb-4d7c-a511-2fe6284c5378\",\"Acanthis flammea flammea\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"Self citation to checklist added. Requires page number.\"\n" + "\"001ee8aa-dbab-43b8-9137-a61565ccf41b\",\"http://phthiraptera.info/node/94211\",\"ee17d179-9f60-4198-ac49-dc9dab3ae529\",\"Brueelia sibirica\",\"\",\"\",\"\",\"ectoparasite of\",\"http://purl.obolibrary.org/obo/RO_0002632\",\"f26a1199-c0bb-4d7c-a511-2fe6284c5378\",\"Acanthis flammea flammea\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"Self citation to checklist added. Requires page number.\"\n" + "\"d0929673-2f4c-49ec-877f-116e74ea360e\",\"http://phthiraptera.info/node/94212\",\"46084bc3-cfbf-4e01-96f8-5ecb50bc5ff9\",\"Ricinus fringillae\",\"\",\"\",\"\",\"ectoparasite of\",\"http://purl.obolibrary.org/obo/RO_0002632\",\"2027cf09-f15d-4c2b-be28-9cb00fabf308\",\"Acanthis flammea\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"\",\"204\",\"[REF: Rheinwald, 1968:204]\"\n";
        return CSVTSVUtil.createLabeledCSVParser(CSVTSVUtil.createExcelCSVParse(IOUtils.toInputStream(firstFewLines, StandardCharsets.UTF_8)));
    };
    final String baseUrl = "https://raw.githubusercontent.com/globalbioticinteractions/natural-history-museum-london-interactions-bank/main";
    final String resource = baseUrl + "/globi.json";
    importAll(interactionListener, tableFactory, baseUrl, resource);
    assertThat(links.size(), is(4));
    for (Map<String, String> firstLine : links) {
        assertNotNull(firstLine.get(DatasetImporterForTSV.INTERACTION_TYPE_NAME));
        assertNotNull(firstLine.get(TaxonUtil.TARGET_TAXON_ID));
        assertNotNull(firstLine.get(TaxonUtil.TARGET_TAXON_NAME));
        assertNotNull(firstLine.get(TaxonUtil.SOURCE_TAXON_ID));
        assertNotNull(firstLine.get(TaxonUtil.SOURCE_TAXON_NAME));
    }
}

Example 39 with Dataset

use of org.globalbioticinteractions.dataset.Dataset in project eol-globi-data by jhpoelen.

the class DatasetImporterForRSSTest method titleIncludeExcludePattern.

@Test
public void titleIncludeExcludePattern() throws StudyImporterException, IOException {
    String configJson = "{ \"url\": \"classpath:/org/eol/globi/data/rss_vertnet.xml\", " + "\"include\": \".*(Arctos).*\", " + "\"exclude\": \".*GGBN.*\", " + "\"hasDependencies\": true }";
    final Dataset dataset = datasetFor(configJson);
    assertFalse(DatasetImporterForRSS.shouldIncludeTitleInDatasetCollection("bla (Arctos) GGBN", dataset));
    assertTrue(DatasetImporterForRSS.shouldIncludeTitleInDatasetCollection("bla (Arctos)", dataset));
    assertFalse(DatasetImporterForRSS.shouldIncludeTitleInDatasetCollection("bla", dataset));
}

Example 40 with Dataset

use of org.globalbioticinteractions.dataset.Dataset in project eol-globi-data by jhpoelen.

the class DatasetImporterForRSSTest method readRSSVertnetWithoutConfig.

@Test
public void readRSSVertnetWithoutConfig() throws StudyImporterException, IOException {
    String configJson = "{ \"url\": \"classpath:/org/eol/globi/data/rss_vertnet.xml\" }";
    final Dataset dataset = datasetFor(configJson);
    List<Dataset> datasets = DatasetImporterForRSS.getDatasetsForFeed(dataset);
    assertThat(datasets.size(), is(263));
    assertThat(datasets.get(0).getOrDefault("hasDependencies", null), is("false"));
    assertThat(datasets.get(0).getOrDefault("url", null), is("http://ipt.vertnet.org:8080/ipt/archive.do?r=utep_mamm"));
    assertThat(datasets.get(0).getArchiveURI(), is(URI.create("http://ipt.vertnet.org:8080/ipt/archive.do?r=utep_mamm")));
}