Search in sources :

Example 1 with FileStorage

use of nl.knaw.huygens.timbuctoo.v5.filestorage.FileStorage in project timbuctoo by HuygensING.

the class ImportManagerTest method makeSimpleDataSet.

@Before
public void makeSimpleDataSet() throws IOException {
    logListLocation = File.createTempFile("logList", ".json");
    logListLocation.delete();
    filesDir = Files.createTempDir();
    fileStorage = new FileSystemFileStorage(filesDir);
    this.importManager = new ImportManager(JsonFileBackedData.getOrCreate(logListLocation, LogList::new, new TypeReference<LogList>() {
    }), fileStorage, fileStorage, fileStorage, Executors.newSingleThreadExecutor(), new Rdf4jIoFactory(), () -> {
    });
}
Also used : FileSystemFileStorage(nl.knaw.huygens.timbuctoo.v5.filestorage.implementations.filesystem.FileSystemFileStorage) LogList(nl.knaw.huygens.timbuctoo.v5.dataset.dto.LogList) Rdf4jIoFactory(nl.knaw.huygens.timbuctoo.v5.rdfio.implementations.rdf4j.Rdf4jIoFactory) Before(org.junit.Before)

Example 2 with FileStorage

use of nl.knaw.huygens.timbuctoo.v5.filestorage.FileStorage in project timbuctoo by HuygensING.

the class DataSet method dataSet.

public static DataSet dataSet(DataSetMetaData metadata, ExecutorService executorService, String rdfPrefix, BdbEnvironmentCreator dataStoreFactory, Runnable onUpdated, DataSetStorage dataSetStorage) throws IOException, DataStoreCreationException {
    String userId = metadata.getOwnerId();
    String dataSetId = metadata.getDataSetId();
    File descriptionFile = dataSetStorage.getResourceSyncDescriptionFile();
    FileStorage fileStorage = dataSetStorage.getFileStorage();
    ImportManager importManager = new ImportManager(dataSetStorage.getLogList(), fileStorage, fileStorage, dataSetStorage.getLogStorage(), executorService, dataSetStorage.getRdfIo(), onUpdated);
    try {
        importManager.subscribeToRdf(new RdfDescriptionSaver(descriptionFile, metadata.getBaseUri(), importManager.getImportStatus()));
    } catch (ParserConfigurationException | SAXException e) {
        LOG.error("Could not construct import manager of data set", e);
    }
    final TupleBinding<String> stringBinding = TupleBinding.getPrimitiveBinding(String.class);
    try {
        StringStringIsCleanHandler stringStringIsCleanHandler = new StringStringIsCleanHandler();
        BdbTripleStore quadStore = new BdbTripleStore(dataStoreFactory.getDatabase(userId, dataSetId, "rdfData", true, stringBinding, stringBinding, stringStringIsCleanHandler));
        final BdbTypeNameStore typeNameStore = new BdbTypeNameStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "typenames", false, stringBinding, stringBinding, stringStringIsCleanHandler)), rdfPrefix);
        final BdbSchemaStore schema = new BdbSchemaStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "schema", false, stringBinding, stringBinding, stringStringIsCleanHandler)), importManager.getImportStatus());
        final BdbTruePatchStore truePatchStore = new BdbTruePatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "truePatch", true, stringBinding, stringBinding, stringStringIsCleanHandler));
        final TupleBinding<Integer> integerBinding = TupleBinding.getPrimitiveBinding(Integer.class);
        final UpdatedPerPatchStore updatedPerPatchStore = new UpdatedPerPatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "updatedPerPatch", true, integerBinding, stringBinding, new IsCleanHandler<Integer, String>() {

            @Override
            public Integer getKey() {
                return Integer.MAX_VALUE;
            }

            @Override
            public String getValue() {
                return "isClean";
            }
        }));
        final BdbRmlDataSourceStore rmlDataSourceStore = new BdbRmlDataSourceStore(dataStoreFactory.getDatabase(userId, dataSetId, "rmlSource", true, stringBinding, stringBinding, stringStringIsCleanHandler), importManager.getImportStatus());
        VersionStore versionStore = new VersionStore(dataStoreFactory.getDatabase(userId, dataSetId, "versions", false, stringBinding, integerBinding, new IsCleanHandler<String, Integer>() {

            @Override
            public String getKey() {
                return "isClean";
            }

            @Override
            public Integer getValue() {
                return Integer.MAX_VALUE;
            }
        }));
        final StoreUpdater storeUpdater = new StoreUpdater(dataStoreFactory, quadStore, typeNameStore, truePatchStore, updatedPerPatchStore, Lists.newArrayList(schema, rmlDataSourceStore), versionStore, importManager.getImportStatus());
        importManager.subscribeToRdf(storeUpdater);
        ImmutableDataSet dataSet = ImmutableDataSet.builder().ownerId(userId).dataSetName(dataSetId).bdbEnvironmentCreator(dataStoreFactory).metadata(metadata).quadStore(quadStore).typeNameStore(typeNameStore).schemaStore(schema).dataSource(new RdfDataSourceFactory(rmlDataSourceStore)).schemaStore(schema).importManager(importManager).dataSetStorage(dataSetStorage).build();
        importManager.init(dataSet);
        if (!quadStore.isClean() || !typeNameStore.isClean() || !schema.isClean() || !truePatchStore.isClean() || !updatedPerPatchStore.isClean() || !rmlDataSourceStore.isClean() || !versionStore.isClean()) {
            LOG.error("Data set '{}__{}' data is corrupted, starting to reimport.", userId, dataSetId);
            quadStore.empty();
            typeNameStore.empty();
            schema.empty();
            truePatchStore.empty();
            updatedPerPatchStore.empty();
            rmlDataSourceStore.empty();
            versionStore.empty();
            importManager.reprocessLogs();
        } else {
            // process unprocessed logs
            importManager.processLogs();
        }
        return dataSet;
    } catch (BdbDbCreationException e) {
        throw new DataStoreCreationException(e.getCause());
    }
}
Also used : ImportManager(nl.knaw.huygens.timbuctoo.v5.dataset.ImportManager) BdbSchemaStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbSchemaStore) BdbTruePatchStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbTruePatchStore) SAXException(org.xml.sax.SAXException) VersionStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.VersionStore) BdbBackedData(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbBackedData) BdbTypeNameStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbTypeNameStore) ParserConfigurationException(javax.xml.parsers.ParserConfigurationException) UpdatedPerPatchStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.UpdatedPerPatchStore) StringStringIsCleanHandler(nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler) RdfDescriptionSaver(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.RdfDescriptionSaver) BdbRmlDataSourceStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbRmlDataSourceStore) StoreUpdater(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.StoreUpdater) IsCleanHandler(nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.IsCleanHandler) StringStringIsCleanHandler(nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler) BdbDbCreationException(nl.knaw.huygens.timbuctoo.v5.berkeleydb.exceptions.BdbDbCreationException) BdbTripleStore(nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbTripleStore) FileStorage(nl.knaw.huygens.timbuctoo.v5.filestorage.FileStorage) RdfDataSourceFactory(nl.knaw.huygens.timbuctoo.v5.rml.RdfDataSourceFactory) DataStoreCreationException(nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.DataStoreCreationException) File(java.io.File)

Example 3 with FileStorage

use of nl.knaw.huygens.timbuctoo.v5.filestorage.FileStorage in project timbuctoo by HuygensING.

the class RsDocumentBuilder method getResourceList.

/**
 * Get the resource list for the dataSet denoted by <code>ownerId</code> and <code>dataSetId</code>.
 * The {@link Optional} is empty if the dataSet is not published and the given <code>user</code> == <code>null</code>
 * or has no read access for the dataSet or the dataSet does not exist.
 *
 * @param user User that requests the list, may be <code>null</code>
 * @param ownerId ownerId
 * @param dataSetId dataSetId
 * @return the resource list for the dataSet denoted by <code>ownerId</code> and <code>dataSetId</code>
 */
public Optional<Urlset> getResourceList(@Nullable User user, String ownerId, String dataSetId) throws IOException {
    Urlset resourceList = null;
    Optional<DataSet> maybeDataSet = dataSetRepository.getDataSet(user, ownerId, dataSetId);
    if (maybeDataSet.isPresent()) {
        DataSetMetaData dataSetMetaData = maybeDataSet.get().getMetadata();
        LogList loglist = maybeDataSet.get().getImportManager().getLogList();
        RsMd rsMd = new RsMd(Capability.RESOURCELIST.xmlValue).withAt(// lastImportDate set on server startup?
        ZonedDateTime.parse(loglist.getLastImportDate()));
        resourceList = new Urlset(rsMd).addLink(new RsLn(REL_UP, rsUriHelper.uriForRsDocument(dataSetMetaData, Capability.CAPABILITYLIST)));
        FileStorage fileStorage = maybeDataSet.get().getFileStorage();
        List<LogEntry> entries = loglist.getEntries();
        entries.sort((e1, e2) -> {
            if (e1.getImportStatus().isPresent() && e2.getImportStatus().isPresent()) {
                return e1.getImportStatus().get().getDate().compareTo(e2.getImportStatus().get().getDate());
            } else if (e1.getImportStatus().isPresent()) {
                return 1;
            } else {
                return -1;
            }
        });
        for (LogEntry logEntry : entries) {
            Optional<String> maybeToken = logEntry.getLogToken();
            if (maybeToken.isPresent()) {
                String loc = rsUriHelper.uriForToken(dataSetMetaData, maybeToken.get());
                Optional<CachedFile> maybeCachedFile = fileStorage.getFile(maybeToken.get());
                if (maybeCachedFile.isPresent()) {
                    UrlItem item = new UrlItem(loc).withMetadata(new RsMd().withType(maybeCachedFile.get().getMimeType().toString()));
                    resourceList.addItem(item);
                }
            }
        }
        rsMd.withCompleted(ZonedDateTime.now(ZoneOffset.UTC));
    }
    return Optional.ofNullable(resourceList);
}
Also used : CachedFile(nl.knaw.huygens.timbuctoo.v5.filestorage.dto.CachedFile) DataSet(nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet) RsLn(nl.knaw.huygens.timbuctoo.remote.rs.xml.RsLn) UrlItem(nl.knaw.huygens.timbuctoo.remote.rs.xml.UrlItem) LogList(nl.knaw.huygens.timbuctoo.v5.dataset.dto.LogList) Urlset(nl.knaw.huygens.timbuctoo.remote.rs.xml.Urlset) FileStorage(nl.knaw.huygens.timbuctoo.v5.filestorage.FileStorage) DataSetMetaData(nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSetMetaData) RsMd(nl.knaw.huygens.timbuctoo.remote.rs.xml.RsMd) LogEntry(nl.knaw.huygens.timbuctoo.v5.dataset.dto.LogEntry)

Aggregations

LogList (nl.knaw.huygens.timbuctoo.v5.dataset.dto.LogList)2 FileStorage (nl.knaw.huygens.timbuctoo.v5.filestorage.FileStorage)2 File (java.io.File)1 ParserConfigurationException (javax.xml.parsers.ParserConfigurationException)1 RsLn (nl.knaw.huygens.timbuctoo.remote.rs.xml.RsLn)1 RsMd (nl.knaw.huygens.timbuctoo.remote.rs.xml.RsMd)1 UrlItem (nl.knaw.huygens.timbuctoo.remote.rs.xml.UrlItem)1 Urlset (nl.knaw.huygens.timbuctoo.remote.rs.xml.Urlset)1 BdbDbCreationException (nl.knaw.huygens.timbuctoo.v5.berkeleydb.exceptions.BdbDbCreationException)1 IsCleanHandler (nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.IsCleanHandler)1 StringStringIsCleanHandler (nl.knaw.huygens.timbuctoo.v5.berkeleydb.isclean.StringStringIsCleanHandler)1 ImportManager (nl.knaw.huygens.timbuctoo.v5.dataset.ImportManager)1 DataSet (nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSet)1 DataSetMetaData (nl.knaw.huygens.timbuctoo.v5.dataset.dto.DataSetMetaData)1 LogEntry (nl.knaw.huygens.timbuctoo.v5.dataset.dto.LogEntry)1 DataStoreCreationException (nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.DataStoreCreationException)1 RdfDescriptionSaver (nl.knaw.huygens.timbuctoo.v5.datastores.implementations.RdfDescriptionSaver)1 BdbBackedData (nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbBackedData)1 BdbRmlDataSourceStore (nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbRmlDataSourceStore)1 BdbSchemaStore (nl.knaw.huygens.timbuctoo.v5.datastores.implementations.bdb.BdbSchemaStore)1