Search in sources :

Example 1 with SchemaStore

use of nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaStore in project timbuctoo by HuygensING.

the class DataSet method dataSet.

public static DataSet dataSet(DataSetMetaData metadata, ExecutorService executorService, String rdfPrefix, BdbEnvironmentCreator dataStoreFactory, Runnable onUpdated, DataSetStorage dataSetStorage) throws IOException, DataStoreCreationException {
    String userId = metadata.getOwnerId();
    String dataSetId = metadata.getDataSetId();
    File descriptionFile = dataSetStorage.getResourceSyncDescriptionFile();
    FileStorage fileStorage = dataSetStorage.getFileStorage();
    ImportManager importManager = new ImportManager(dataSetStorage.getLogList(), fileStorage, fileStorage, dataSetStorage.getLogStorage(), executorService, dataSetStorage.getRdfIo(), onUpdated);
    try {
        importManager.subscribeToRdf(new RdfDescriptionSaver(descriptionFile, metadata.getBaseUri(), importManager.getImportStatus()));
    } catch (ParserConfigurationException | SAXException e) {
        LOG.error("Could not construct import manager of data set", e);
    }
    final TupleBinding<String> stringBinding = TupleBinding.getPrimitiveBinding(String.class);
    try {
        StringStringIsCleanHandler stringStringIsCleanHandler = new StringStringIsCleanHandler();
        BdbTripleStore quadStore = new BdbTripleStore(dataStoreFactory.getDatabase(userId, dataSetId, "rdfData", true, stringBinding, stringBinding, stringStringIsCleanHandler));
        final BdbTypeNameStore typeNameStore = new BdbTypeNameStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "typenames", false, stringBinding, stringBinding, stringStringIsCleanHandler)), rdfPrefix);
        final BdbSchemaStore schema = new BdbSchemaStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "schema", false, stringBinding, stringBinding, stringStringIsCleanHandler)), importManager.getImportStatus());
        final BdbTruePatchStore truePatchStore = new BdbTruePatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "truePatch", true, stringBinding, stringBinding, stringStringIsCleanHandler));
        final TupleBinding<Integer> integerBinding = TupleBinding.getPrimitiveBinding(Integer.class);
        final UpdatedPerPatchStore updatedPerPatchStore = new UpdatedPerPatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "updatedPerPatch", true, integerBinding, stringBinding, new IsCleanHandler<Integer, String>() {

            @Override
            public Integer getKey() {
                return Integer.MAX_VALUE;
            }

            @Override
            public String getValue() {
                return "isClean";
            }
        }));
        final BdbRmlDataSourceStore rmlDataSourceStore = new BdbRmlDataSourceStore(dataStoreFactory.getDatabase(userId, dataSetId, "rmlSource", true, stringBinding, stringBinding, stringStringIsCleanHandler), importManager.getImportStatus());
        VersionStore versionStore = new VersionStore(dataStoreFactory.getDatabase(userId, dataSetId, "versions", false, stringBinding, integerBinding, new IsCleanHandler<String, Integer>() {

            @Override
            public String getKey() {
                return "isClean";
            }

            @Override
            public Integer getValue() {
                return Integer.MAX_VALUE;
            }
        }));
        final StoreUpdater storeUpdater = new StoreUpdater(dataStoreFactory, quadStore, typeNameStore, truePatchStore, updatedPerPatchStore, Lists.newArrayList(schema, rmlDataSourceStore), versionStore, importManager.getImportStatus());
        importManager.subscribeToRdf(storeUpdater);
        ImmutableDataSet dataSet = ImmutableDataSet.builder().ownerId(userId).dataSetName(dataSetId).bdbEnvironmentCreator(dataStoreFactory).metadata(metadata).quadStore(quadStore).typeNameStore(typeNameStore).schemaStore(schema).dataSource(new RdfDataSourceFactory(rmlDataSourceStore)).schemaStore(schema).importManager(importManager).dataSetStorage(dataSetStorage).build();
        importManager.init(dataSet);
        if (!quadStore.isClean() || !typeNameStore.isClean() || !schema.isClean() || !truePatchStore.isClean() || !updatedPerPatchStore.isClean() || !rmlDataSourceStore.isClean() || !versionStore.isClean()) {
            LOG.error("Data set '{}__{}' data is corrupted, starting to reimport.", userId, dataSetId);
            quadStore.empty();
            typeNameStore.empty();
            schema.empty();
            truePatchStore.empty();
            updatedPerPatchStore.empty();
            rmlDataSourceStore.empty();
            versionStore.empty();
            importManager.reprocessLogs();
        } else {
            // process unprocessed logs
            importManager.processLogs();
        }
        return dataSet;
    } catch (BdbDbCreationException e) {
        throw new DataStoreCreationException(e.getCause());
    }
}
Also used : ImportManager(nl.knaw.huygens.timbuctoo.v5.dataset.ImportManager) BdbSchemaStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbSchemaStore) BdbTruePatchStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbTruePatchStore) SAXException(org.xml.sax.SAXException) VersionStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.VersionStore) BdbBackedData(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbBackedData) BdbTypeNameStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbTypeNameStore) ParserConfigurationException(javax.xml.parsers.ParserConfigurationException) UpdatedPerPatchStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.UpdatedPerPatchStore) StringStringIsCleanHandler(nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler) RdfDescriptionSaver(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.RdfDescriptionSaver) BdbRmlDataSourceStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbRmlDataSourceStore) StoreUpdater(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.StoreUpdater) IsCleanHandler(nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.IsCleanHandler) StringStringIsCleanHandler(nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler) BdbDbCreationException(nl.knaw.huygens.timbuctoo.v5.berkeleydb.exceptions.BdbDbCreationException) BdbTripleStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbTripleStore) FileStorage(nl.knaw.huygens.timbuctoo.v5.filestorage.FileStorage) RdfDataSourceFactory(nl.knaw.huygens.timbuctoo.v5.rml.RdfDataSourceFactory) DataStoreCreationException(nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.DataStoreCreationException) File(java.io.File)

Example 2 with SchemaStore

use of nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaStore in project timbuctoo by HuygensING.

the class ExtendSchemaMutation method get.

@Override
public Object get(DataFetchingEnvironment env) {
    DataSet dataSet = MutationHelpers.getDataSet(env, dataSetRepository::getDataSet);
    MutationHelpers.checkAdminPermissions(env, dataSet.getMetadata());
    final SchemaStore generatedSchema = dataSet.getSchemaStore();
    Map<String, Type> customTypes = new HashMap<>();
    List<ExplicitType> customSchema;
    Map<String, List<ExplicitField>> newCustomSchema = new HashMap<>();
    try {
        String customSchemaString = OBJECT_MAPPER.writeValueAsString(env.getArgument("customSchema"));
        customSchema = OBJECT_MAPPER.readValue(customSchemaString, new TypeReference<List<ExplicitType>>() {
        });
    } catch (IOException e) {
        throw new RuntimeException("Could not parse the schema input");
    }
    for (ExplicitType explicitType : customSchema) {
        customTypes.put(explicitType.getCollectionId(), explicitType.convertToType());
        newCustomSchema.put(explicitType.getCollectionId(), explicitType.getFields());
    }
    Map<String, List<ExplicitField>> existingCustomSchema = dataSet.getCustomSchema();
    Map<String, Type> existingCustomSchemaTypes = new HashMap<>();
    for (Map.Entry<String, List<ExplicitField>> entry : existingCustomSchema.entrySet()) {
        ExplicitType tempExplicitType = new ExplicitType(entry.getKey(), entry.getValue());
        existingCustomSchemaTypes.put(entry.getKey(), tempExplicitType.convertToType());
    }
    MergeSchemas mergeSchemas = new MergeSchemas();
    customTypes = mergeSchemas.mergeSchema(existingCustomSchemaTypes, customTypes);
    MergeExplicitSchemas mergeExplicitSchemas = new MergeExplicitSchemas();
    Map<String, List<ExplicitField>> mergedExplicitSchema = mergeExplicitSchemas.mergeExplicitSchemas(existingCustomSchema, newCustomSchema);
    try {
        dataSet.saveCustomSchema(mergedExplicitSchema);
    } catch (IOException e) {
        LOG.error("Saving the custom schema failed", e);
        throw new RuntimeException(e);
    }
    Map<String, Type> mergedSchema = mergeSchemas.mergeSchema(generatedSchema.getStableTypes(), customTypes);
    for (Map.Entry<String, Type> customType : customTypes.entrySet()) {
        if (!mergedSchema.containsKey(customType.getKey())) {
            return ImmutableMap.of("message", "Schema extension was unsuccessful.");
        }
    }
    return ImmutableMap.of("message", "Schema extended successfully.");
}
Also used : MergeSchemas(nl.knaw.huygens.timbuctoo.v5.graphql.customschema.MergeSchemas) DataSet(nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet) SchemaStore(nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaStore) HashMap(java.util.HashMap) MergeExplicitSchemas(nl.knaw.huygens.timbuctoo.v5.graphql.customschema.MergeExplicitSchemas) IOException(java.io.IOException) ExplicitType(nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.dto.ExplicitType) Type(nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.dto.Type) List(java.util.List) TypeReference(com.fasterxml.jackson.core.type.TypeReference) ImmutableMap(jersey.repackaged.com.google.common.collect.ImmutableMap) HashMap(java.util.HashMap) Map(java.util.Map) ExplicitType(nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.dto.ExplicitType)

Example 3 with SchemaStore

use of nl.knaw.huygens.timbuctoo.v5.datastores.schemastore.SchemaStore in project timbuctoo by HuygensING.

the class BdbSchemaStoreTest method generatesASchemaForAllAdded.

@Test
public void generatesASchemaForAllAdded() throws Exception {
    final DummyDataStorage dataStore = new DummyDataStorage();
    ChangeFetcher changeFetcher = new DummyChangeFetcher(CursorQuad.create("subj", "pred", Direction.OUT, ChangeType.ASSERTED, "obj", null, null, ""), CursorQuad.create("subj", RdfConstants.RDF_TYPE, Direction.OUT, ChangeType.ASSERTED, "type", null, null, ""), CursorQuad.create("obj", "pred", Direction.IN, ChangeType.ASSERTED, "subj", null, null, ""));
    BdbSchemaStore schemaStore = new BdbSchemaStore(dataStore, new ImportStatus(new LogList()));
    schemaStore.start();
    schemaStore.onChangedSubject("subj", changeFetcher);
    schemaStore.onChangedSubject("obj", changeFetcher);
    schemaStore.finish();
    assertThat(dataStore.getResult(), is("{\n" + "  \"http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#unknown\" : {\n" + "    \"name\" : \"http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#unknown\",\n" + "    \"predicates\" : [ {\n" + "      \"name\" : \"pred\",\n" + "      \"direction\" : \"IN\",\n" + "      \"valueTypes\" : { },\n" + "      \"referenceTypes\" : {\n" + "        \"type\" : 1\n" + "      },\n" + "      \"subjectsWithThisPredicate\" : 1,\n" + "      \"subjectsWithThisPredicateAsList\" : 0,\n" + "      \"hasBeenList\" : false,\n" + "      \"hasBeenSingular\" : true\n" + "    } ],\n" + "    \"subjectsWithThisType\" : 1\n" + "  },\n" + "  \"type\" : {\n" + "    \"name\" : \"type\",\n" + "    \"predicates\" : [ {\n" + "      \"name\" : \"http://www.w3.org/1999/02/22-rdf-syntax-ns#type\",\n" + "      \"direction\" : \"OUT\",\n" + "      \"valueTypes\" : { },\n" + "      \"referenceTypes\" : {\n" + "        \"http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#unknown\" : 1\n" + "      },\n" + "      \"subjectsWithThisPredicate\" : 1,\n" + "      \"subjectsWithThisPredicateAsList\" : 0,\n" + "      \"hasBeenList\" : false,\n" + "      \"hasBeenSingular\" : true\n" + "    }, {\n" + "      \"name\" : \"pred\",\n" + "      \"direction\" : \"OUT\",\n" + "      \"valueTypes\" : { },\n" + "      \"referenceTypes\" : {\n" + "        \"http://timbuctoo.huygens.knaw.nl/static/v5/vocabulary#unknown\" : 1\n" + "      },\n" + "      \"subjectsWithThisPredicate\" : 1,\n" + "      \"subjectsWithThisPredicateAsList\" : 0,\n" + "      \"hasBeenList\" : false,\n" + "      \"hasBeenSingular\" : true\n" + "    } ],\n" + "    \"subjectsWithThisType\" : 1\n" + "  }\n" + "}"));
}
Also used : LogList(nl.knaw.huygens.timbuctoo.v5.dataset.dto.LogList) ImportStatus(nl.knaw.huygens.timbuctoo.v5.dataset.ImportStatus) ChangeFetcher(nl.knaw.huygens.timbuctoo.v5.dataset.ChangeFetcher) Test(org.junit.Test)

Aggregations

TypeReference (com.fasterxml.jackson.core.type.TypeReference)1 File (java.io.File)1 IOException (java.io.IOException)1 HashMap (java.util.HashMap)1 List (java.util.List)1 Map (java.util.Map)1 ParserConfigurationException (javax.xml.parsers.ParserConfigurationException)1 ImmutableMap (jersey.repackaged.com.google.common.collect.ImmutableMap)1 BdbDbCreationException (nl.knaw.huygens.timbuctoo.v5.berkeleydb.exceptions.BdbDbCreationException)1 IsCleanHandler (nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.IsCleanHandler)1 StringStringIsCleanHandler (nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler)1 ChangeFetcher (nl.knaw.huygens.timbuctoo.v5.dataset.ChangeFetcher)1 ImportManager (nl.knaw.huygens.timbuctoo.v5.dataset.ImportManager)1 ImportStatus (nl.knaw.huygens.timbuctoo.v5.dataset.ImportStatus)1 DataSet (nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet)1 LogList (nl.knaw.huygens.timbuctoo.v5.dataset.dto.LogList)1 DataStoreCreationException (nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.DataStoreCreationException)1 RdfDescriptionSaver (nl.knaw.huygens.timbuctoo.v5.datastores.implementations.RdfDescriptionSaver)1 BdbBackedData (nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbBackedData)1 BdbRmlDataSourceStore (nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbRmlDataSourceStore)1