Examples with DataSet nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet used on opensource projects

Example 21 with DataSet

use of nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet in project timbuctoo by HuygensING.

the class PermissionBasedFieldVisibilityTest method getFieldDefinitionReturnsFieldDefinitionIfNotDataSetField.

@Test
public void getFieldDefinitionReturnsFieldDefinitionIfNotDataSetField() throws Exception {
    final DataSetRepository dataSetRepository = mock(DataSetRepository.class);
    DataSet dataSet = createDataSetWithUserPermissions("user__dataSetUserHasAccessTo", Sets.newHashSet(Permission.READ));
    DataSet dataSet2 = createDataSetWithUserPermissions("user__dataSetUserDoesNotHasAccessTo", Sets.newHashSet());
    Collection<DataSet> dataSetCollection = Sets.newHashSet(dataSet, dataSet2);
    given(dataSetRepository.getDataSets()).willReturn(dataSetCollection);
    final PermissionBasedFieldVisibility permissionBasedFieldVisibility = new PermissionBasedFieldVisibility(userPermissionCheck, dataSetRepository);
    final GraphQLFieldsContainer graphQlFieldsContainer = createGraphQlFieldsContainer("user__dataSetUserHasAccessTo", "user__dataSetUserDoesNotHasAccessTo", "nonDataSetField");
    GraphQLFieldDefinition retrievedGraphQlFieldDefinition = permissionBasedFieldVisibility.getFieldDefinition(graphQlFieldsContainer, // new String to make sure the
    new String("nonDataSetField"));
    // contents are compared not the instance.
    assertThat(retrievedGraphQlFieldDefinition, hasProperty("name", is("nonDataSetField")));
}

Example 22 with DataSet

use of nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet in project timbuctoo by HuygensING.

the class RmlRdfCreator method sendQuads.

@Override
public void sendQuads(RdfSerializer saver, DataSet dataSet, Consumer<String> status) throws LogStorageFailedException {
    RdfDataSourceFactory dataSourceFactory = dataSet.getDataSource();
    final Model model = ModelFactory.createDefaultModel();
    try {
        model.read(new ByteArrayInputStream(rdfData.getBytes(StandardCharsets.UTF_8)), null, "JSON-LD");
    } catch (Exception e) {
        throw new LogStorageFailedException(e);
    }
    final RmlMappingDocument rmlMappingDocument = rmlBuilder.fromRdf(model, dataSourceFactory::apply);
    if (rmlMappingDocument.getErrors().size() > 0) {
        throw new LogStorageFailedException("failure: " + String.join("\nfailure: ", rmlMappingDocument.getErrors()) + "\n");
    }
    // FIXME: trigger onprefix for all rml prefixes
    // FIXME: store rml and retrieve it from tripleStore when mapping
    Stream<Quad> triples = rmlMappingDocument.execute(new ReportingErrorHandler(status));
    Iterator<Quad> iterator = triples.iterator();
    while (iterator.hasNext()) {
        Quad triple = iterator.next();
        saver.onQuad(triple.getSubject().getUri().get(), triple.getPredicate().getUri().get(), triple.getObject().getContent(), triple.getObject().getLiteralType().orElse(null), triple.getObject().getLiteralLanguage().orElse(null), baseUri);
    }
}

Example 23 with DataSet

use of nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet in project timbuctoo by HuygensING.

the class DataSetRepository method publishDataSet.

public void publishDataSet(User user, String ownerId, String dataSetName) throws DataSetPublishException {
    Optional<DataSet> dataSet = getDataSet(user, ownerId, dataSetName);
    try {
        if (dataSet.isPresent() && permissionFetcher.getPermissions(user, dataSet.get().getMetadata()).contains(Permission.ADMIN)) {
            DataSetMetaData dataSetMetaData = dataSet.get().getMetadata();
            dataSetMetaData.publish();
            try {
                dataStorage.getDataSetStorage(ownerId, dataSetName).saveMetaData(dataSetMetaData);
            } catch (DataStorageSaveException e) {
                throw new DataSetPublishException(e);
            }
        }
    } catch (PermissionFetchingException e) {
        throw new DataSetPublishException(e);
    }
}

Example 24 with DataSet

use of nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet in project timbuctoo by HuygensING.

the class DataSetRepository method createDataSet.

public DataSet createDataSet(User user, String dataSetId) throws DataStoreCreationException, IllegalDataSetNameException {
    // The ownerId might not be valid (i.e. a safe string). We make it safe here:
    // dataSetId is under the control of the user so we simply throw if it's not valid
    String ownerPrefix = "u" + user.getPersistentId();
    final String baseUri = rdfIdHelper.dataSetBaseUri(ownerPrefix, dataSetId);
    String uriPrefix;
    if (!baseUri.endsWith("/") && !baseUri.endsWith("#") && !baseUri.endsWith("?")) {
        // #boo&foo=bar
        if (baseUri.contains("#") || baseUri.contains("?")) {
            if (baseUri.endsWith("&")) {
                uriPrefix = baseUri;
            } else {
                uriPrefix = baseUri + "&";
            }
        } else {
            uriPrefix = baseUri + "/";
        }
    } else {
        uriPrefix = baseUri;
    }
    final DataSetMetaData dataSet = new BasicDataSetMetaData(ownerPrefix, dataSetId, baseUri, uriPrefix, false, publicByDefault);
    try {
        dataStorage.getDataSetStorage(ownerPrefix, dataSetId).saveMetaData(dataSet);
    } catch (DataStorageSaveException e) {
        throw new DataStoreCreationException(e);
    }
    synchronized (dataSetMap) {
        Map<String, DataSet> userDataSets = dataSetMap.computeIfAbsent(ownerPrefix, key -> new HashMap<>());
        if (!userDataSets.containsKey(dataSetId)) {
            try {
                permissionFetcher.initializeOwnerAuthorization(user, dataSet.getOwnerId(), dataSet.getDataSetId());
                userDataSets.put(dataSetId, dataSet(dataSet, executorService, rdfBaseUri, dataStoreFactory, () -> onUpdated.accept(dataSet.getCombinedId()), dataStorage.getDataSetStorage(ownerPrefix, dataSetId)));
            } catch (PermissionFetchingException | AuthorizationCreationException | IOException e) {
                throw new DataStoreCreationException(e);
            }
        }
        return userDataSets.get(dataSetId);
    }
}

Example 25 with DataSet

use of nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet in project timbuctoo by HuygensING.

the class DataSet method dataSet.

public static DataSet dataSet(DataSetMetaData metadata, ExecutorService executorService, String rdfPrefix, BdbEnvironmentCreator dataStoreFactory, Runnable onUpdated, DataSetStorage dataSetStorage) throws IOException, DataStoreCreationException {
    String userId = metadata.getOwnerId();
    String dataSetId = metadata.getDataSetId();
    File descriptionFile = dataSetStorage.getResourceSyncDescriptionFile();
    FileStorage fileStorage = dataSetStorage.getFileStorage();
    ImportManager importManager = new ImportManager(dataSetStorage.getLogList(), fileStorage, fileStorage, dataSetStorage.getLogStorage(), executorService, dataSetStorage.getRdfIo(), onUpdated);
    try {
        importManager.subscribeToRdf(new RdfDescriptionSaver(descriptionFile, metadata.getBaseUri(), importManager.getImportStatus()));
    } catch (ParserConfigurationException | SAXException e) {
        LOG.error("Could not construct import manager of data set", e);
    }
    final TupleBinding<String> stringBinding = TupleBinding.getPrimitiveBinding(String.class);
    try {
        StringStringIsCleanHandler stringStringIsCleanHandler = new StringStringIsCleanHandler();
        BdbTripleStore quadStore = new BdbTripleStore(dataStoreFactory.getDatabase(userId, dataSetId, "rdfData", true, stringBinding, stringBinding, stringStringIsCleanHandler));
        final BdbTypeNameStore typeNameStore = new BdbTypeNameStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "typenames", false, stringBinding, stringBinding, stringStringIsCleanHandler)), rdfPrefix);
        final BdbSchemaStore schema = new BdbSchemaStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "schema", false, stringBinding, stringBinding, stringStringIsCleanHandler)), importManager.getImportStatus());
        final BdbTruePatchStore truePatchStore = new BdbTruePatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "truePatch", true, stringBinding, stringBinding, stringStringIsCleanHandler));
        final TupleBinding<Integer> integerBinding = TupleBinding.getPrimitiveBinding(Integer.class);
        final UpdatedPerPatchStore updatedPerPatchStore = new UpdatedPerPatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "updatedPerPatch", true, integerBinding, stringBinding, new IsCleanHandler<Integer, String>() {

            @Override
            public Integer getKey() {
                return Integer.MAX_VALUE;
            }

            @Override
            public String getValue() {
                return "isClean";
            }
        }));
        final BdbRmlDataSourceStore rmlDataSourceStore = new BdbRmlDataSourceStore(dataStoreFactory.getDatabase(userId, dataSetId, "rmlSource", true, stringBinding, stringBinding, stringStringIsCleanHandler), importManager.getImportStatus());
        VersionStore versionStore = new VersionStore(dataStoreFactory.getDatabase(userId, dataSetId, "versions", false, stringBinding, integerBinding, new IsCleanHandler<String, Integer>() {

            @Override
            public String getKey() {
                return "isClean";
            }

            @Override
            public Integer getValue() {
                return Integer.MAX_VALUE;
            }
        }));
        final StoreUpdater storeUpdater = new StoreUpdater(dataStoreFactory, quadStore, typeNameStore, truePatchStore, updatedPerPatchStore, Lists.newArrayList(schema, rmlDataSourceStore), versionStore, importManager.getImportStatus());
        importManager.subscribeToRdf(storeUpdater);
        ImmutableDataSet dataSet = ImmutableDataSet.builder().ownerId(userId).dataSetName(dataSetId).bdbEnvironmentCreator(dataStoreFactory).metadata(metadata).quadStore(quadStore).typeNameStore(typeNameStore).schemaStore(schema).dataSource(new RdfDataSourceFactory(rmlDataSourceStore)).schemaStore(schema).importManager(importManager).dataSetStorage(dataSetStorage).build();
        importManager.init(dataSet);
        if (!quadStore.isClean() || !typeNameStore.isClean() || !schema.isClean() || !truePatchStore.isClean() || !updatedPerPatchStore.isClean() || !rmlDataSourceStore.isClean() || !versionStore.isClean()) {
            LOG.error("Data set '{}__{}' data is corrupted, starting to reimport.", userId, dataSetId);
            quadStore.empty();
            typeNameStore.empty();
            schema.empty();
            truePatchStore.empty();
            updatedPerPatchStore.empty();
            rmlDataSourceStore.empty();
            versionStore.empty();
            importManager.reprocessLogs();
        } else {
            // process unprocessed logs
            importManager.processLogs();
        }
        return dataSet;
    } catch (BdbDbCreationException e) {
        throw new DataStoreCreationException(e.getCause());
    }
}