Examples with ImportManager nl.knaw.huygens.timbuctoo.v5.dataset.ImportManager used on opensource projects

Example 1 with ImportManager

use of nl.knaw.huygens.timbuctoo.v5.dataset.ImportManager in project timbuctoo by HuygensING.

the class Rml method upload.

@POST
public Response upload(final String rdfData, @PathParam("userId") final String ownerId, @PathParam("dataSetId") final String dataSetId, @HeaderParam("authorization") String authHeader) throws DataStoreCreationException, LogStorageFailedException, ExecutionException, InterruptedException {
    Optional<User> user;
    try {
        user = userValidator.getUserFromAccessToken(authHeader);
    } catch (UserValidationException e) {
        LOG.error("Exception validating user", e);
        return Response.status(Response.Status.UNAUTHORIZED).build();
    }
    if (!user.isPresent()) {
        return Response.status(Response.Status.UNAUTHORIZED).build();
    }
    final Optional<DataSet> dataSet = dataSetRepository.getDataSet(user.get(), ownerId, dataSetId);
    if (dataSet.isPresent()) {
        ImportManager importManager = dataSet.get().getImportManager();
        final String baseUri = dataSet.get().getMetadata().getBaseUri();
        Future<ImportStatus> promise = importManager.generateLog(baseUri, baseUri, new RmlRdfCreator(baseUri, rdfData));
        return handleImportManagerResult(promise);
    } else {
        return errorResponseHelper.dataSetNotFound(ownerId, dataSetId);
    }
}

Example 2 with ImportManager

use of nl.knaw.huygens.timbuctoo.v5.dataset.ImportManager in project timbuctoo by HuygensING.

the class TabularUpload method upload.

@Consumes(MediaType.MULTIPART_FORM_DATA)
@Produces(MediaType.APPLICATION_JSON)
@POST
public Response upload(@FormDataParam("file") final InputStream rdfInputStream, @FormDataParam("file") final FormDataBodyPart body, @FormDataParam("file") final FormDataContentDisposition fileInfo, @FormDataParam("fileMimeTypeOverride") final MediaType mimeTypeOverride, FormDataMultiPart formData, @HeaderParam("authorization") final String authHeader, @PathParam("userId") final String ownerId, @PathParam("dataSetId") final String dataSetId, @QueryParam("forceCreation") boolean forceCreation) throws DataStoreCreationException, FileStorageFailedException, ExecutionException, InterruptedException, LogStorageFailedException {
    final Either<Response, Response> result = authCheck.getOrCreate(authHeader, ownerId, dataSetId, forceCreation).flatMap(userAndDs -> authCheck.hasAdminAccess(userAndDs.getLeft(), userAndDs.getRight())).map(userAndDs -> {
        final MediaType mediaType = mimeTypeOverride == null ? body.getMediaType() : mimeTypeOverride;
        Optional<Loader> loader = LoaderFactory.createFor(mediaType.toString(), formData.getFields().entrySet().stream().filter(entry -> entry.getValue().size() > 0).filter(entry -> entry.getValue().get(0) != null).filter(entry -> MediaTypes.typeEqual(MediaType.TEXT_PLAIN_TYPE, entry.getValue().get(0).getMediaType())).collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().get(0).getValue())));
        if (!loader.isPresent()) {
            return errorResponseHelper.error(400, "We do not support the mediatype '" + mediaType + "'. Make sure to add the correct mediatype to the file " + "parameter. In curl you'd use `-F \"file=@<filename>;type=<mediatype>\"`. In a webbrowser you probably " + "have no way of setting the correct mimetype. So you can use a special parameter to override it: " + "`formData.append(\"fileMimeTypeOverride\", \"<mimetype>\");`");
        }
        final DataSet dataSet = userAndDs.getRight();
        ImportManager importManager = dataSet.getImportManager();
        if (StringUtils.isBlank(fileInfo.getName())) {
            return Response.status(400).entity("filename cannot be empty.").build();
        }
        try {
            String fileToken = importManager.addFile(rdfInputStream, fileInfo.getFileName(), mediaType);
            Future<ImportStatus> promise = importManager.generateLog(dataSet.getMetadata().getBaseUri(), dataSet.getMetadata().getBaseUri(), new TabularRdfCreator(loader.get(), fileToken, fileInfo.getFileName()));
            return handleImportManagerResult(promise);
        } catch (FileStorageFailedException | LogStorageFailedException e) {
            LOG.error("Tabular upload failed", e);
            return Response.serverError().build();
        }
    });
    if (result.isLeft()) {
        return result.getLeft();
    } else {
        return result.get();
    }
}

Example 3 with ImportManager

use of nl.knaw.huygens.timbuctoo.v5.dataset.ImportManager in project timbuctoo by HuygensING.

the class ImportManagerTest method makeSimpleDataSet.

@Before
public void makeSimpleDataSet() throws IOException {
    logListLocation = File.createTempFile("logList", ".json");
    logListLocation.delete();
    filesDir = Files.createTempDir();
    fileStorage = new FileSystemFileStorage(filesDir);
    this.importManager = new ImportManager(JsonFileBackedData.getOrCreate(logListLocation, LogList::new, new TypeReference<LogList>() {
    }), fileStorage, fileStorage, fileStorage, Executors.newSingleThreadExecutor(), new Rdf4jIoFactory(), () -> {
    });
}

Example 4 with ImportManager

use of nl.knaw.huygens.timbuctoo.v5.dataset.ImportManager in project timbuctoo by HuygensING.

the class DataSet method dataSet.

public static DataSet dataSet(DataSetMetaData metadata, ExecutorService executorService, String rdfPrefix, BdbEnvironmentCreator dataStoreFactory, Runnable onUpdated, DataSetStorage dataSetStorage) throws IOException, DataStoreCreationException {
    String userId = metadata.getOwnerId();
    String dataSetId = metadata.getDataSetId();
    File descriptionFile = dataSetStorage.getResourceSyncDescriptionFile();
    FileStorage fileStorage = dataSetStorage.getFileStorage();
    ImportManager importManager = new ImportManager(dataSetStorage.getLogList(), fileStorage, fileStorage, dataSetStorage.getLogStorage(), executorService, dataSetStorage.getRdfIo(), onUpdated);
    try {
        importManager.subscribeToRdf(new RdfDescriptionSaver(descriptionFile, metadata.getBaseUri(), importManager.getImportStatus()));
    } catch (ParserConfigurationException | SAXException e) {
        LOG.error("Could not construct import manager of data set", e);
    }
    final TupleBinding<String> stringBinding = TupleBinding.getPrimitiveBinding(String.class);
    try {
        StringStringIsCleanHandler stringStringIsCleanHandler = new StringStringIsCleanHandler();
        BdbTripleStore quadStore = new BdbTripleStore(dataStoreFactory.getDatabase(userId, dataSetId, "rdfData", true, stringBinding, stringBinding, stringStringIsCleanHandler));
        final BdbTypeNameStore typeNameStore = new BdbTypeNameStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "typenames", false, stringBinding, stringBinding, stringStringIsCleanHandler)), rdfPrefix);
        final BdbSchemaStore schema = new BdbSchemaStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "schema", false, stringBinding, stringBinding, stringStringIsCleanHandler)), importManager.getImportStatus());
        final BdbTruePatchStore truePatchStore = new BdbTruePatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "truePatch", true, stringBinding, stringBinding, stringStringIsCleanHandler));
        final TupleBinding<Integer> integerBinding = TupleBinding.getPrimitiveBinding(Integer.class);
        final UpdatedPerPatchStore updatedPerPatchStore = new UpdatedPerPatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "updatedPerPatch", true, integerBinding, stringBinding, new IsCleanHandler<Integer, String>() {

            @Override
            public Integer getKey() {
                return Integer.MAX_VALUE;
            }

            @Override
            public String getValue() {
                return "isClean";
            }
        }));
        final BdbRmlDataSourceStore rmlDataSourceStore = new BdbRmlDataSourceStore(dataStoreFactory.getDatabase(userId, dataSetId, "rmlSource", true, stringBinding, stringBinding, stringStringIsCleanHandler), importManager.getImportStatus());
        VersionStore versionStore = new VersionStore(dataStoreFactory.getDatabase(userId, dataSetId, "versions", false, stringBinding, integerBinding, new IsCleanHandler<String, Integer>() {

            @Override
            public String getKey() {
                return "isClean";
            }

            @Override
            public Integer getValue() {
                return Integer.MAX_VALUE;
            }
        }));
        final StoreUpdater storeUpdater = new StoreUpdater(dataStoreFactory, quadStore, typeNameStore, truePatchStore, updatedPerPatchStore, Lists.newArrayList(schema, rmlDataSourceStore), versionStore, importManager.getImportStatus());
        importManager.subscribeToRdf(storeUpdater);
        ImmutableDataSet dataSet = ImmutableDataSet.builder().ownerId(userId).dataSetName(dataSetId).bdbEnvironmentCreator(dataStoreFactory).metadata(metadata).quadStore(quadStore).typeNameStore(typeNameStore).schemaStore(schema).dataSource(new RdfDataSourceFactory(rmlDataSourceStore)).schemaStore(schema).importManager(importManager).dataSetStorage(dataSetStorage).build();
        importManager.init(dataSet);
        if (!quadStore.isClean() || !typeNameStore.isClean() || !schema.isClean() || !truePatchStore.isClean() || !updatedPerPatchStore.isClean() || !rmlDataSourceStore.isClean() || !versionStore.isClean()) {
            LOG.error("Data set '{}__{}' data is corrupted, starting to reimport.", userId, dataSetId);
            quadStore.empty();
            typeNameStore.empty();
            schema.empty();
            truePatchStore.empty();
            updatedPerPatchStore.empty();
            rmlDataSourceStore.empty();
            versionStore.empty();
            importManager.reprocessLogs();
        } else {
            // process unprocessed logs
            importManager.processLogs();
        }
        return dataSet;
    } catch (BdbDbCreationException e) {
        throw new DataStoreCreationException(e.getCause());
    }
}

Example 5 with ImportManager

use of nl.knaw.huygens.timbuctoo.v5.dataset.ImportManager in project timbuctoo by HuygensING.

the class JsonLdEditEndpoint method submitChanges.

@PUT
public Response submitChanges(String jsonLdImport, @PathParam("user") String ownerId, @PathParam("dataset") String dataSetId, @HeaderParam("authorization") String authHeader) throws LogStorageFailedException {
    Optional<User> user;
    try {
        user = userValidator.getUserFromAccessToken(authHeader);
    } catch (UserValidationException e) {
        user = Optional.empty();
    }
    Optional<DataSet> dataSetOpt = dataSetRepository.getDataSet(user.get(), ownerId, dataSetId);
    if (!dataSetOpt.isPresent()) {
        return Response.status(Response.Status.NOT_FOUND).build();
    }
    final DataSet dataSet = dataSetOpt.get();
    final QuadStore quadStore = dataSet.getQuadStore();
    final ImportManager importManager = dataSet.getImportManager();
    final Response response = checkWriteAccess(dataSet, user, permissionFetcher);
    if (response != null) {
        return response;
    }
    try {
        final Future<ImportStatus> promise = importManager.generateLog(dataSet.getMetadata().getBaseUri(), dataSet.getMetadata().getBaseUri(), fromCurrentState(documentLoader, jsonLdImport, quadStore, TIM_USERS + user.get().getPersistentId(), UUID.randomUUID().toString(), Clock.systemUTC()));
        return handleImportManagerResult(promise);
    } catch (IOException e) {
        return Response.status(Response.Status.BAD_REQUEST).entity(e.getMessage()).build();
    } catch (ConcurrentUpdateException e) {
        return Response.status(Response.Status.CONFLICT).entity(e.getMessage()).build();
    }
}