use of nl.knaw.huygens.timbuctoo.v5.filestorage.FileStorage in project timbuctoo by HuygensING.
the class ImportManagerTest method makeSimpleDataSet.
@Before
public void makeSimpleDataSet() throws IOException {
logListLocation = File.createTempFile("logList", ".json");
logListLocation.delete();
filesDir = Files.createTempDir();
fileStorage = new FileSystemFileStorage(filesDir);
this.importManager = new ImportManager(JsonFileBackedData.getOrCreate(logListLocation, LogList::new, new TypeReference<LogList>() {
}), fileStorage, fileStorage, fileStorage, Executors.newSingleThreadExecutor(), new Rdf4jIoFactory(), () -> {
});
}
use of nl.knaw.huygens.timbuctoo.v5.filestorage.FileStorage in project timbuctoo by HuygensING.
the class DataSet method dataSet.
public static DataSet dataSet(DataSetMetaData metadata, ExecutorService executorService, String rdfPrefix, BdbEnvironmentCreator dataStoreFactory, Runnable onUpdated, DataSetStorage dataSetStorage) throws IOException, DataStoreCreationException {
String userId = metadata.getOwnerId();
String dataSetId = metadata.getDataSetId();
File descriptionFile = dataSetStorage.getResourceSyncDescriptionFile();
FileStorage fileStorage = dataSetStorage.getFileStorage();
ImportManager importManager = new ImportManager(dataSetStorage.getLogList(), fileStorage, fileStorage, dataSetStorage.getLogStorage(), executorService, dataSetStorage.getRdfIo(), onUpdated);
try {
importManager.subscribeToRdf(new RdfDescriptionSaver(descriptionFile, metadata.getBaseUri(), importManager.getImportStatus()));
} catch (ParserConfigurationException | SAXException e) {
LOG.error("Could not construct import manager of data set", e);
}
final TupleBinding<String> stringBinding = TupleBinding.getPrimitiveBinding(String.class);
try {
StringStringIsCleanHandler stringStringIsCleanHandler = new StringStringIsCleanHandler();
BdbTripleStore quadStore = new BdbTripleStore(dataStoreFactory.getDatabase(userId, dataSetId, "rdfData", true, stringBinding, stringBinding, stringStringIsCleanHandler));
final BdbTypeNameStore typeNameStore = new BdbTypeNameStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "typenames", false, stringBinding, stringBinding, stringStringIsCleanHandler)), rdfPrefix);
final BdbSchemaStore schema = new BdbSchemaStore(new BdbBackedData(dataStoreFactory.getDatabase(userId, dataSetId, "schema", false, stringBinding, stringBinding, stringStringIsCleanHandler)), importManager.getImportStatus());
final BdbTruePatchStore truePatchStore = new BdbTruePatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "truePatch", true, stringBinding, stringBinding, stringStringIsCleanHandler));
final TupleBinding<Integer> integerBinding = TupleBinding.getPrimitiveBinding(Integer.class);
final UpdatedPerPatchStore updatedPerPatchStore = new UpdatedPerPatchStore(dataStoreFactory.getDatabase(userId, dataSetId, "updatedPerPatch", true, integerBinding, stringBinding, new IsCleanHandler<Integer, String>() {
@Override
public Integer getKey() {
return Integer.MAX_VALUE;
}
@Override
public String getValue() {
return "isClean";
}
}));
final BdbRmlDataSourceStore rmlDataSourceStore = new BdbRmlDataSourceStore(dataStoreFactory.getDatabase(userId, dataSetId, "rmlSource", true, stringBinding, stringBinding, stringStringIsCleanHandler), importManager.getImportStatus());
VersionStore versionStore = new VersionStore(dataStoreFactory.getDatabase(userId, dataSetId, "versions", false, stringBinding, integerBinding, new IsCleanHandler<String, Integer>() {
@Override
public String getKey() {
return "isClean";
}
@Override
public Integer getValue() {
return Integer.MAX_VALUE;
}
}));
final StoreUpdater storeUpdater = new StoreUpdater(dataStoreFactory, quadStore, typeNameStore, truePatchStore, updatedPerPatchStore, Lists.newArrayList(schema, rmlDataSourceStore), versionStore, importManager.getImportStatus());
importManager.subscribeToRdf(storeUpdater);
ImmutableDataSet dataSet = ImmutableDataSet.builder().ownerId(userId).dataSetName(dataSetId).bdbEnvironmentCreator(dataStoreFactory).metadata(metadata).quadStore(quadStore).typeNameStore(typeNameStore).schemaStore(schema).dataSource(new RdfDataSourceFactory(rmlDataSourceStore)).schemaStore(schema).importManager(importManager).dataSetStorage(dataSetStorage).build();
importManager.init(dataSet);
if (!quadStore.isClean() || !typeNameStore.isClean() || !schema.isClean() || !truePatchStore.isClean() || !updatedPerPatchStore.isClean() || !rmlDataSourceStore.isClean() || !versionStore.isClean()) {
LOG.error("Data set '{}__{}' data is corrupted, starting to reimport.", userId, dataSetId);
quadStore.empty();
typeNameStore.empty();
schema.empty();
truePatchStore.empty();
updatedPerPatchStore.empty();
rmlDataSourceStore.empty();
versionStore.empty();
importManager.reprocessLogs();
} else {
// process unprocessed logs
importManager.processLogs();
}
return dataSet;
} catch (BdbDbCreationException e) {
throw new DataStoreCreationException(e.getCause());
}
}
use of nl.knaw.huygens.timbuctoo.v5.filestorage.FileStorage in project timbuctoo by HuygensING.
the class RsDocumentBuilder method getResourceList.
/**
* Get the resource list for the dataSet denoted by <code>ownerId</code> and <code>dataSetId</code>.
* The {@link Optional} is empty if the dataSet is not published and the given <code>user</code> == <code>null</code>
* or has no read access for the dataSet or the dataSet does not exist.
*
* @param user User that requests the list, may be <code>null</code>
* @param ownerId ownerId
* @param dataSetId dataSetId
* @return the resource list for the dataSet denoted by <code>ownerId</code> and <code>dataSetId</code>
*/
public Optional<Urlset> getResourceList(@Nullable User user, String ownerId, String dataSetId) throws IOException {
Urlset resourceList = null;
Optional<DataSet> maybeDataSet = dataSetRepository.getDataSet(user, ownerId, dataSetId);
if (maybeDataSet.isPresent()) {
DataSetMetaData dataSetMetaData = maybeDataSet.get().getMetadata();
LogList loglist = maybeDataSet.get().getImportManager().getLogList();
RsMd rsMd = new RsMd(Capability.RESOURCELIST.xmlValue).withAt(// lastImportDate set on server startup?
ZonedDateTime.parse(loglist.getLastImportDate()));
resourceList = new Urlset(rsMd).addLink(new RsLn(REL_UP, rsUriHelper.uriForRsDocument(dataSetMetaData, Capability.CAPABILITYLIST)));
FileStorage fileStorage = maybeDataSet.get().getFileStorage();
List<LogEntry> entries = loglist.getEntries();
entries.sort((e1, e2) -> {
if (e1.getImportStatus().isPresent() && e2.getImportStatus().isPresent()) {
return e1.getImportStatus().get().getDate().compareTo(e2.getImportStatus().get().getDate());
} else if (e1.getImportStatus().isPresent()) {
return 1;
} else {
return -1;
}
});
for (LogEntry logEntry : entries) {
Optional<String> maybeToken = logEntry.getLogToken();
if (maybeToken.isPresent()) {
String loc = rsUriHelper.uriForToken(dataSetMetaData, maybeToken.get());
Optional<CachedFile> maybeCachedFile = fileStorage.getFile(maybeToken.get());
if (maybeCachedFile.isPresent()) {
UrlItem item = new UrlItem(loc).withMetadata(new RsMd().withType(maybeCachedFile.get().getMimeType().toString()));
resourceList.addItem(item);
}
}
}
rsMd.withCompleted(ZonedDateTime.now(ZoneOffset.UTC));
}
return Optional.ofNullable(resourceList);
}
Aggregations