Examples with TinkerPopOperations nl.knaw.huygens.timbuctoo.database.tinkerpop.TinkerPopOperations used on opensource projects

Example 1 with TinkerPopOperations

use of nl.knaw.huygens.timbuctoo.database.tinkerpop.TinkerPopOperations in project timbuctoo by HuygensING.

the class TripleImporterIntegrationTest method setUp.

@Before
public void setUp() throws Exception {
    // FIXME create stub
    // Timbuctoo needs at least an Admin VRE.
    Vres vres = getVres();
    graphWrapper = newGraph().wrap();
    TinkerPopOperations tinkerPopOperations = new TinkerPopOperations(graphWrapper);
    tinkerPopOperations.saveVre(vres.getVre("Admin"));
    timbuctooActions = TimbuctooActionsStubs.withDataStore(tinkerPopOperations);
    rdfImportSession = RdfImportSession.cleanImportSession(VRE_NAME, tinkerPopOperations, new EntityFinisherHelper());
    instance = new TripleImporter(graphWrapper, VRE_NAME, rdfImportSession);
}

Example 2 with TinkerPopOperations

use of nl.knaw.huygens.timbuctoo.database.tinkerpop.TinkerPopOperations in project timbuctoo by HuygensING.

the class TinkerPopOperationsForMigrationsTest method itGIvesYouAnInitDb.

@Test
public void itGIvesYouAnInitDb() throws JsonProcessingException {
    final TinkerPopGraphManager graphManager = newGraph().wrap();
    TinkerPopOperations ops = forInitDb(graphManager);
    assertThat(ops.databaseIsEmptyExceptForMigrations(), is(true));
    ops.initDb(new VresBuilder().withVre("Admin", "", vre -> vre.withCollection("relations", CollectionBuilder::isRelationCollection)).build(), relationType("concept", "hasFirstPerson", "person", "someRel", false, false, false, UUID.randomUUID()));
    assertThat(graphManager.getGraph().traversal().V().count().next(), is(4L));
}

Example 3 with TinkerPopOperations

use of nl.knaw.huygens.timbuctoo.database.tinkerpop.TinkerPopOperations in project timbuctoo by HuygensING.

the class TimbuctooV4 method run.

@Override
public void run(TimbuctooConfiguration configuration, Environment environment) throws Exception {
    // environment.jersey().property(ServerProperties.TRACING, "ALL");
    // // environment.jersey().property(ServerProperties.TRACING_THRESHOLD, "VERBOSE");
    // Make sure we know what version is running
    Properties properties = new Properties();
    InputStream gitproperties = getClass().getClassLoader().getResourceAsStream("git.properties");
    String currentVersion;
    if (gitproperties != null) {
        properties.load(gitproperties);
        currentVersion = properties.getProperty("git.commit.id");
    } else {
        currentVersion = "NO-GIT-PROPERTIES-FOUND";
        LoggerFactory.getLogger(this.getClass()).error("NO-GIT-PROPERTIES-FOUND");
    }
    LoggerFactory.getLogger(this.getClass()).info("Now launching timbuctoo version: " + currentVersion);
    HttpClientBuilder apacheHttpClientBuilder = new HttpClientBuilder(environment).using(configuration.getHttpClientConfiguration());
    CloseableHttpClient httpClient = apacheHttpClientBuilder.build("httpclient");
    // Support services
    SecurityFactory securityConfig = configuration.getSecurityConfiguration().createNewSecurityFactory(httpClient);
    securityConfig.getHealthChecks().forEachRemaining(check -> {
        register(environment, check.getLeft(), new LambdaHealthCheck(check.getRight()));
    });
    // Database migration
    LinkedHashMap<String, DatabaseMigration> migrations = new LinkedHashMap<>();
    migrations.put("fix-dcarkeywords-displayname-migration", new FixDcarKeywordDisplayNameMigration());
    migrations.put("fix-pids-migration", new MakePidsAbsoluteUrls());
    UriHelper uriHelper = configuration.getUriHelper();
    environment.lifecycle().addServerLifecycleListener(new BaseUriDeriver(configuration));
    TinkerPopConfig tinkerPopConfig = configuration.getDatabaseConfiguration();
    final TinkerPopGraphManager graphManager = new TinkerPopGraphManager(tinkerPopConfig, migrations);
    final PersistenceManager persistenceManager = configuration.getPersistenceManagerFactory().build();
    UrlGenerator uriToRedirectToFromPersistentUrls = (coll, id, rev) -> uriHelper.fromResourceUri(SingleEntity.makeUrl(coll, id, rev));
    final UrlGenerator pathWithoutVersionAndRevision = (coll, id, rev) -> URI.create(SingleEntity.makeUrl(coll, id, null).toString().replaceFirst("^/v2.1/", ""));
    final UrlGenerator uriWithoutRev = (coll, id, rev) -> uriHelper.fromResourceUri(SingleEntity.makeUrl(coll, id, null));
    HandleAdder handleAdder = new HandleAdder(persistenceManager, activeMqBundle);
    // TODO make function when TimbuctooActions does not depend on TransactionEnforcer anymore
    TimbuctooActions.TimbuctooActionsFactory timbuctooActionsFactory = new TimbuctooActions.TimbuctooActionsFactoryImpl(securityConfig.getPermissionFetcher(), Clock.systemDefaultZone(), handleAdder, uriToRedirectToFromPersistentUrls, () -> new TinkerPopOperations(graphManager));
    TransactionEnforcer transactionEnforcer = new TransactionEnforcer(timbuctooActionsFactory);
    graphManager.onGraph(g -> new ScaffoldMigrator(graphManager).execute());
    handleAdder.init(transactionEnforcer);
    final Vres vres = new DatabaseConfiguredVres(transactionEnforcer);
    migrations.put("prepare-for-bia-import-migration", new PrepareForBiaImportMigration(vres, graphManager));
    migrations.put("give-existing-relationtypes-rdf-uris", new RelationTypeRdfUriMigration());
    migrations.put("remove-search-results", new RemoveSearchResultsMigration());
    migrations.put("move-indices-to-isLatest-vertex", new MoveIndicesToIsLatestVertexMigration(vres));
    final ResourceSyncService resourceSyncService = new ResourceSyncService(httpClient, new ResourceSyncContext());
    final JsonMetadata jsonMetadata = new JsonMetadata(vres, graphManager);
    final AutocompleteService.AutocompleteServiceFactory autocompleteServiceFactory = new AutocompleteService.AutocompleteServiceFactory(uriWithoutRev);
    environment.lifecycle().manage(graphManager);
    final CrudServiceFactory crudServiceFactory = new CrudServiceFactory(vres, securityConfig.getUserValidator(), pathWithoutVersionAndRevision);
    final Webhooks webhooks = configuration.getWebhooks().getWebHook(environment);
    DataSetRepository dataSetRepository = configuration.getDataSetConfiguration().createRepository(environment.lifecycle().executorService("dataSet").build(), securityConfig.getPermissionFetcher(), configuration.getDatabases(), configuration.getRdfIdHelper(), (combinedId -> {
        try {
            webhooks.dataSetUpdated(combinedId);
        } catch (IOException e) {
            LOG.error("Webhook call failed", e);
        }
    }), configuration.dataSetsArePublicByDefault());
    environment.lifecycle().manage(new DataSetRepositoryManager(dataSetRepository));
    ErrorResponseHelper errorResponseHelper = new ErrorResponseHelper();
    AuthCheck authCheck = new AuthCheck(securityConfig.getUserValidator(), securityConfig.getPermissionFetcher(), dataSetRepository);
    register(environment, new RdfUpload(authCheck));
    register(environment, new TabularUpload(authCheck, dataSetRepository, errorResponseHelper));
    register(environment, new Rml(dataSetRepository, errorResponseHelper, securityConfig.getUserValidator()));
    SerializerWriterRegistry serializerWriterRegistry = new SerializerWriterRegistry(new CsvWriter(), new JsonLdWriter(), new JsonWriter(), new GraphVizWriter());
    final PaginationArgumentsHelper argHelper = new PaginationArgumentsHelper(configuration.getCollectionFilters());
    final GraphQl graphQlEndpoint = new GraphQl(new RootQuery(dataSetRepository, serializerWriterRegistry, configuration.getArchetypesSchema(), new RdfWiringFactory(dataSetRepository, argHelper, configuration.getDefaultSummaryProps()), new DerivedSchemaTypeGenerator(argHelper), environment.getObjectMapper()), serializerWriterRegistry, securityConfig.getUserValidator(), uriHelper, securityConfig.getPermissionFetcher(), dataSetRepository);
    register(environment, graphQlEndpoint);
    if (securityConfig instanceof TwitterSecurityFactory) {
        final TwitterLogin twitterLogin = new TwitterLogin();
        register(environment, twitterLogin);
    }
    register(environment, new JsonLdEditEndpoint(securityConfig.getUserValidator(), securityConfig.getPermissionFetcher(), dataSetRepository, new HttpClientBuilder(environment).build("json-ld")));
    register(environment, new RootEndpoint(uriHelper, configuration.getUserRedirectUrl()));
    if (securityConfig instanceof OldStyleSecurityFactory) {
        register(environment, new Authenticate(((OldStyleSecurityFactory) securityConfig).getLoggedInUsers()));
    }
    register(environment, new Me(securityConfig.getUserValidator()));
    register(environment, new Search(configuration, uriHelper, graphManager));
    register(environment, new Autocomplete(autocompleteServiceFactory, transactionEnforcer));
    register(environment, new Index(securityConfig.getUserValidator(), crudServiceFactory, transactionEnforcer));
    register(environment, new SingleEntity(securityConfig.getUserValidator(), crudServiceFactory, transactionEnforcer));
    register(environment, new SingleEntityNTriple(transactionEnforcer, uriHelper));
    register(environment, new WomenWritersEntityGet(crudServiceFactory, transactionEnforcer));
    register(environment, new LegacySingleEntityRedirect(uriHelper));
    register(environment, new LegacyIndexRedirect(uriHelper));
    register(environment, new Discover(resourceSyncService));
    if (configuration.isAllowGremlinEndpoint()) {
        register(environment, new Gremlin(graphManager));
    }
    register(environment, new Graph(graphManager, vres));
    register(environment, new RelationTypes(graphManager));
    register(environment, new Metadata());
    register(environment, new nl.knaw.huygens.timbuctoo.server.endpoints.v2.system.vres.Metadata(jsonMetadata));
    register(environment, new MyVres(securityConfig.getUserValidator(), securityConfig.getPermissionFetcher(), transactionEnforcer, uriHelper));
    register(environment, new ListVres(uriHelper, transactionEnforcer));
    register(environment, new VreImage(transactionEnforcer));
    final ExecutorService rfdExecutorService = environment.lifecycle().executorService("rdf-import").build();
    register(environment, new ImportRdf(graphManager, vres, rfdExecutorService, transactionEnforcer));
    register(environment, new Import(new ResourceSyncFileLoader(httpClient), authCheck));
    register(environment, new WellKnown());
    RsDocumentBuilder rsDocumentBuilder = new RsDocumentBuilder(dataSetRepository, configuration.getUriHelper());
    register(environment, new RsEndpoint(rsDocumentBuilder, securityConfig.getUserValidator()));
    // Admin resources
    if (securityConfig instanceof OldStyleSecurityFactory) {
        final OldStyleSecurityFactory oldStyleSecurityFactory = (OldStyleSecurityFactory) securityConfig;
        environment.admin().addTask(new UserCreationTask(new LocalUserCreator(oldStyleSecurityFactory.getLoginCreator(), oldStyleSecurityFactory.getUserCreator(), oldStyleSecurityFactory.getVreAuthorizationCreator())));
    }
    environment.admin().addTask(new DatabaseValidationTask(new DatabaseValidator(graphManager, new LabelsAddedToVertexDatabaseCheck(), new InvariantsCheck(vres), new FullTextIndexCheck()), Clock.systemUTC(), 5000));
    environment.admin().addTask(new DbLogCreatorTask(graphManager));
    environment.admin().addTask(new BdbDumpTask(configuration.getDatabases()));
    if (configuration.getDatabaseBackupper().isPresent()) {
        environment.admin().addTask(new StagingBackup(configuration.getDatabaseBackupper().get().create(configuration.getDatabaseConfiguration().getDatabasePath(), configuration.getDatabases().getDatabaseLocation())));
    }
    // register health checks
    // Dropwizard Health checks are used to check whether requests should be routed to this instance
    // For example, checking if neo4j is in a valid state is not a "HealthCheck" because if the database on one instance
    // is in an invalid state, then this applies to all other instances too. So once the database is in an invalid state
    // timbuctoo will be down.
    // 
    // checking whether this instance is part of the neo4j quorum is a good HealthCheck because running a database query
    // on those instances that are not part of the quorum will block forever, while the other instances will respond
    // just fine.
    register(environment, "Neo4j database connection", graphManager);
    // Log all http requests
    register(environment, new LoggingFilter(1024, currentVersion));
    register(environment, new TransactionFilter(graphManager));
    // Allow all CORS requests
    register(environment, new PromiscuousCorsFilter());
    // Add embedded AMQ (if any) to the metrics
    configuration.getLocalAmqJmxPath(HANDLE_QUEUE).ifPresent(rethrowConsumer(jmxPath -> {
        String dwMetricName = name(this.getClass(), "localAmq");
        ObjectName jmxMetricName = new ObjectName(jmxPath);
        environment.metrics().register(dwMetricName + ".enqueueCount", new JmxAttributeGauge(jmxMetricName, "EnqueueCount"));
        environment.metrics().register(dwMetricName + ".dequeueCount", new JmxAttributeGauge(jmxMetricName, "DequeueCount"));
    }));
    setupObjectMapping(environment);
}

Example 4 with TinkerPopOperations

use of nl.knaw.huygens.timbuctoo.database.tinkerpop.TinkerPopOperations in project timbuctoo by HuygensING.

the class ScaffoldMigrator method execute.

// FIXME move to DataAccess (allow ScaffoldVresConfig.mappings to be injected as an argument)
public void execute() {
    // The code below will add vertices, so a second launch will not run this code
    try (TinkerPopOperations db = forInitDb(graphManager)) {
        if (db.databaseIsEmptyExceptForMigrations()) {
            LOG.info("Setting up a new scaffold for empty database");
            Vres mappings = new VresBuilder().withVre("Admin", "", vre -> vre.withCollection("persons", collection -> collection.withDescription("People with at least a name, birthdate and birthplace.").withDisplayName(localProperty("person_names", defaultFullPersonNameConverter)).withProperty("names", localProperty("person_names", personNames)).withProperty("gender", localProperty("person_gender")).withProperty("birthDate", localProperty("person_birthDate", datable)).withProperty("deathDate", localProperty("person_deathDate", datable))).withCollection("locations", collection -> collection.withDescription("Countries, cities, villages, streets, etc.").withDisplayName(localProperty("location_name")).withProperty("name", localProperty("location_name")).withProperty("country", localProperty("location_country"))).withCollection("collectives", collection -> collection.withDescription("Institutes, multiple persons, companies, etc.").withDisplayName(localProperty("collective_name")).withProperty("name", localProperty("collective_name"))).withCollection("documents", collection -> collection.withDescription("Stories, novels, letters, diaries, plays, films, etc.").withDisplayName(localProperty("document_title")).withProperty("title", localProperty("document_title")).withProperty("documentType", localProperty("document_documentType")).withProperty("date", localProperty("document_date", datable))).withCollection("keywords", collection -> collection.withDescription("").withDisplayName(localProperty("keyword_value")).withProperty("value", localProperty("keyword_value")).withProperty("type", localProperty("keyword_type"))).withCollection("concepts", collection -> collection.withDescription("Concepts which do not conform to a specific archetype.").withDisplayName(localProperty("label"))).withCollection("relations", CollectionBuilder::isRelationCollection)).build();
            db.initDb(mappings, relationType("person", "hasBirthPlace", "location", "isBirthPlaceOf", false, false, false, UUID.randomUUID()), relationType("person", "hasDeathPlace", "location", "isDeathPlaceOf", false, false, false, UUID.randomUUID()), relationType("collective", "hasMember", "person", "isMemberOf", false, false, false, UUID.randomUUID()), relationType("collective", "locatedAt", "location", "isHomeOf", false, false, false, UUID.randomUUID()), relationType("document", "isCreatedBy", "person", "isCreatorOf", false, false, false, UUID.randomUUID()), // person to person relations
            relationType("concept", "hasFirstPerson", "person", "isFirstPersonInRelation", false, false, false, UUID.randomUUID()), relationType("concept", "hasSecondPerson", "person", "isSecondPersonInRelation", false, false, false, UUID.randomUUID()), relationType("concept", "hasPersonToPersonRelationType", "concept", "isPersonToPersonRelationTypeOf", false, false, false, UUID.randomUUID()), // states of persons
            relationType("concept", "hasStateType", "concept", "isStateTypeOf", false, false, false, UUID.randomUUID()), relationType("concept", "isStateOfPerson", "person", "hasPersonState", false, false, false, UUID.randomUUID()), relationType("concept", "isStateLinkedToInstitute", "collective", "isInstituteLinkedToState", false, false, false, UUID.randomUUID()), relationType("concept", "isStateLinkedToLocation", "location", "isLocationLinkedToState", false, false, false, UUID.randomUUID()), // data lines for persons
            relationType("concept", "hasDataLineType", "concept", "isDataLineTypeOf", false, false, false, UUID.randomUUID()), relationType("concept", "isDataLineForPerson", "person", "hasDataLine", false, false, false, UUID.randomUUID()), // scientist_bios for persons
            relationType("concept", "hasFieldOfInterest", "concept", "isFieldOfInterestOf", false, false, false, UUID.randomUUID()), relationType("concept", "isScientistBioOf", "person", "hasScientistBio", false, false, false, UUID.randomUUID()));
        }
        db.success();
    }
}

Example 5 with TinkerPopOperations

use of nl.knaw.huygens.timbuctoo.database.tinkerpop.TinkerPopOperations in project timbuctoo by HuygensING.

the class TinkerPopOperationsForMigrations method saveRelationTypes.

static void saveRelationTypes(TinkerPopGraphManager graphManager, RelationType... relationTypes) {
    try (TinkerPopOperations dataStoreOperations = new TinkerPopOperations(graphManager)) {
        dataStoreOperations.saveRelationTypes(relationTypes);
        dataStoreOperations.success();
    }
}