use of nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException in project timbuctoo by HuygensING.
the class ImportManager method processLogsUntil.
private synchronized ImportStatus processLogsUntil(int maxIndex) {
importStatus.start(this.getClass().getSimpleName() + ".processLogs", null);
ListIterator<LogEntry> unprocessed = logListStore.getData().getUnprocessed();
boolean dataWasAdded = false;
while (unprocessed.hasNext() && unprocessed.nextIndex() <= maxIndex) {
int index = unprocessed.nextIndex();
LogEntry entry = unprocessed.next();
importStatus.startEntry(entry);
if (entry.getLogToken().isPresent()) {
// logToken
String logToken = entry.getLogToken().get();
try (CachedLog log = logStorage.getLog(logToken)) {
final Stopwatch stopwatch = Stopwatch.createStarted();
for (RdfProcessor processor : subscribedProcessors) {
if (processor.getCurrentVersion() <= index) {
String msg = "******* " + processor.getClass().getSimpleName() + " Started importing full log...";
LOG.info(msg);
importStatus.setStatus(msg);
RdfParser rdfParser = serializerFactory.makeRdfParser(log);
processor.start(index);
rdfParser.importRdf(log, entry.getBaseUri(), entry.getDefaultGraph(), processor);
processor.commit();
}
}
long elapsedTime = stopwatch.elapsed(TimeUnit.SECONDS);
String msg = "Finished importing. Total import took " + elapsedTime + " seconds.";
LOG.info(msg);
importStatus.setStatus(msg);
dataWasAdded = true;
} catch (Exception e) {
LOG.error("Processing log failed", e);
importStatus.addError("Processing log failed", e);
}
// Update the log, even after RdfProcessingFailedException | IOException
try {
logListStore.updateData(logList -> {
logList.markAsProcessed(index);
return logList;
});
} catch (IOException e) {
LOG.error("Updating the log failed", e);
importStatus.addError("Updating log failed", e);
}
} else {
// no logToken
RdfCreator creator = entry.getRdfCreator().get();
String token = "";
MediaType mediaType;
Optional<Charset> charset;
File tempFile = null;
try {
tempFile = File.createTempFile("log_to_generate", "nq");
try (OutputStream stream = new GZIPOutputStream(new FileOutputStream(tempFile))) {
if (creator instanceof PlainRdfCreator) {
try (RdfSerializer serializer = serializerFactory.makeRdfSerializer(stream)) {
mediaType = serializer.getMediaType();
charset = Optional.of(serializer.getCharset());
((PlainRdfCreator) creator).sendQuads(serializer, dataSet, importStatus::setStatus);
} catch (Exception e) {
LOG.error("Log generation failed", e);
importStatus.addError("Log generation failed", e);
break;
}
} else {
try (RdfPatchSerializer srlzr = serializerFactory.makeRdfPatchSerializer(stream, entry.getBaseUri())) {
mediaType = srlzr.getMediaType();
charset = Optional.of(srlzr.getCharset());
((PatchRdfCreator) creator).sendQuads(srlzr, importStatus::setStatus, dataSet);
} catch (Exception e) {
LOG.error("Log generation failed", e);
importStatus.addError("Log generation failed", e);
break;
}
}
}
try (InputStream inputStream = new GZIPInputStream(new FileInputStream(tempFile))) {
token = logStorage.saveLog(inputStream, "log_generated_by_" + creator.getClass().getSimpleName(), mediaType, charset);
}
LogEntry entryWithLog;
entryWithLog = LogEntry.addLogToEntry(entry, token);
unprocessed.set(entryWithLog);
token = "";
// move back to process this item again
unprocessed.previous();
} catch (Exception e) {
if (token.isEmpty()) {
LOG.error("Log processing failed", e);
} else {
LOG.error("Log processing failed. Log created but not added to the list!", e);
}
importStatus.addError("Log processing failed", e);
break;
} finally {
if (tempFile != null) {
tempFile.delete();
}
}
}
// end else with no condition
importStatus.finishEntry();
}
// end main while loop
if (dataWasAdded) {
webhooks.run();
}
importStatus.finishList();
// update log.json
try {
logListStore.updateData(Function.identity());
} catch (IOException e) {
LOG.error("Updating the log failed", e);
importStatus.addError("Updating log failed", e);
}
return importStatus;
}
use of nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException in project timbuctoo by HuygensING.
the class RdfDescriptionSaver method delValue.
@Override
public void delValue(String subject, String predicate, String value, String dataType, String graph) throws RdfProcessingFailedException {
try {
if (Objects.equals(subject, baseUri)) {
ValueFactory vf = SimpleValueFactory.getInstance();
model.remove(vf.createIRI(subject), vf.createIRI(predicate), vf.createLiteral(value));
}
} catch (Exception e) {
throw new RdfProcessingFailedException(e);
}
}
use of nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException in project timbuctoo by HuygensING.
the class RdfDescriptionSaver method addRelation.
@Override
public void addRelation(String subject, String predicate, String object, String graph) throws RdfProcessingFailedException {
try {
if (Objects.equals(subject, baseUri) && isDescriptionPredicate(predicate)) {
ValueFactory vf = SimpleValueFactory.getInstance();
model.add(vf.createIRI(subject), vf.createIRI(predicate), vf.createIRI(object));
}
} catch (Exception e) {
throw new RdfProcessingFailedException(e);
}
}
use of nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException in project timbuctoo by HuygensING.
the class BdbRmlDataSourceStore method onChangedSubject.
@Override
public void onChangedSubject(String subject, ChangeFetcher changeFetcher) throws RdfProcessingFailedException {
try {
boolean[] wasCollection = new boolean[] { false };
StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, TIM_HAS_ROW, Direction.OUT, true, false, true), quad -> {
wasCollection[0] = true;
if (quad.getChangeType() == ChangeType.ASSERTED) {
bdbWrapper.put(subject, quad.getObject());
}
});
if (!wasCollection[0]) {
try (Stream<CursorQuad> quads = changeFetcher.getPredicates(subject, TIM_HAS_ROW, Direction.IN, true, true, true)) {
Optional<CursorQuad> isRawRow = quads.findFirst();
if (isRawRow.isPresent()) {
final String collectionUri = isRawRow.get().getObject();
Map<String, Property> predicatesToStore = collectionProperties.computeIfAbsent(collectionUri, collection -> getPropertyNames(changeFetcher, collection));
switch(isRawRow.get().getChangeType()) {
case ASSERTED:
// add all unchanged and new predicates
StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, false, true, true), pred -> {
final Property property = predicatesToStore.get(pred.getPredicate());
if (property != null) {
bdbWrapper.put(subject, property.newName + "\n" + pred.getObject());
}
});
break;
case RETRACTED:
// remove all unchanged and removed predicates
StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, true, true, false), pred -> {
final Property property = predicatesToStore.get(pred.getPredicate());
if (property != null) {
bdbWrapper.delete(subject, property.oldName + "\n" + pred.getObject());
}
});
break;
case UNCHANGED:
// add all added predicates, remove all removed predicates
StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, true, false, true), pred -> {
final Property property = predicatesToStore.get(pred.getPredicate());
if (property != null) {
if (pred.getChangeType() == ChangeType.RETRACTED) {
bdbWrapper.delete(subject, property.oldName + "\n" + pred.getObject());
} else {
bdbWrapper.put(subject, property.oldName + "\n" + pred.getObject());
}
}
});
break;
default:
throw new RuntimeException("Should not happen");
}
}
}
}
} catch (DatabaseWriteException e) {
throw new RdfProcessingFailedException(e);
}
}
use of nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException in project timbuctoo by HuygensING.
the class RdfDescriptionSaver method delRelation.
@Override
public void delRelation(String subject, String predicate, String object, String graph) throws RdfProcessingFailedException {
try {
if (Objects.equals(subject, baseUri)) {
ValueFactory vf = SimpleValueFactory.getInstance();
model.remove(vf.createIRI(subject), vf.createIRI(predicate), vf.createIRI(object));
}
} catch (Exception e) {
throw new RdfProcessingFailedException(e);
}
}
Aggregations