Examples with RdfPatchSerializer nl.knaw.huygens.timbuctoo.v5.rdfio.RdfPatchSerializer used on opensource projects

Example 1 with RdfPatchSerializer

use of nl.knaw.huygens.timbuctoo.v5.rdfio.RdfPatchSerializer in project timbuctoo by HuygensING.

the class ImportManager method processLogsUntil.

private synchronized ImportStatus processLogsUntil(int maxIndex) {
    importStatus.start(this.getClass().getSimpleName() + ".processLogs", null);
    ListIterator<LogEntry> unprocessed = logListStore.getData().getUnprocessed();
    boolean dataWasAdded = false;
    while (unprocessed.hasNext() && unprocessed.nextIndex() <= maxIndex) {
        int index = unprocessed.nextIndex();
        LogEntry entry = unprocessed.next();
        importStatus.startEntry(entry);
        if (entry.getLogToken().isPresent()) {
            // logToken
            String logToken = entry.getLogToken().get();
            try (CachedLog log = logStorage.getLog(logToken)) {
                final Stopwatch stopwatch = Stopwatch.createStarted();
                for (RdfProcessor processor : subscribedProcessors) {
                    if (processor.getCurrentVersion() <= index) {
                        String msg = "******* " + processor.getClass().getSimpleName() + " Started importing full log...";
                        LOG.info(msg);
                        importStatus.setStatus(msg);
                        RdfParser rdfParser = serializerFactory.makeRdfParser(log);
                        processor.start(index);
                        rdfParser.importRdf(log, entry.getBaseUri(), entry.getDefaultGraph(), processor);
                        processor.commit();
                    }
                }
                long elapsedTime = stopwatch.elapsed(TimeUnit.SECONDS);
                String msg = "Finished importing. Total import took " + elapsedTime + " seconds.";
                LOG.info(msg);
                importStatus.setStatus(msg);
                dataWasAdded = true;
            } catch (Exception e) {
                LOG.error("Processing log failed", e);
                importStatus.addError("Processing log failed", e);
            }
            // Update the log, even after RdfProcessingFailedException | IOException
            try {
                logListStore.updateData(logList -> {
                    logList.markAsProcessed(index);
                    return logList;
                });
            } catch (IOException e) {
                LOG.error("Updating the log failed", e);
                importStatus.addError("Updating log failed", e);
            }
        } else {
            // no logToken
            RdfCreator creator = entry.getRdfCreator().get();
            String token = "";
            MediaType mediaType;
            Optional<Charset> charset;
            File tempFile = null;
            try {
                tempFile = File.createTempFile("log_to_generate", "nq");
                try (OutputStream stream = new GZIPOutputStream(new FileOutputStream(tempFile))) {
                    if (creator instanceof PlainRdfCreator) {
                        try (RdfSerializer serializer = serializerFactory.makeRdfSerializer(stream)) {
                            mediaType = serializer.getMediaType();
                            charset = Optional.of(serializer.getCharset());
                            ((PlainRdfCreator) creator).sendQuads(serializer, dataSet, importStatus::setStatus);
                        } catch (Exception e) {
                            LOG.error("Log generation failed", e);
                            importStatus.addError("Log generation failed", e);
                            break;
                        }
                    } else {
                        try (RdfPatchSerializer srlzr = serializerFactory.makeRdfPatchSerializer(stream, entry.getBaseUri())) {
                            mediaType = srlzr.getMediaType();
                            charset = Optional.of(srlzr.getCharset());
                            ((PatchRdfCreator) creator).sendQuads(srlzr, importStatus::setStatus, dataSet);
                        } catch (Exception e) {
                            LOG.error("Log generation failed", e);
                            importStatus.addError("Log generation failed", e);
                            break;
                        }
                    }
                }
                try (InputStream inputStream = new GZIPInputStream(new FileInputStream(tempFile))) {
                    token = logStorage.saveLog(inputStream, "log_generated_by_" + creator.getClass().getSimpleName(), mediaType, charset);
                }
                LogEntry entryWithLog;
                entryWithLog = LogEntry.addLogToEntry(entry, token);
                unprocessed.set(entryWithLog);
                token = "";
                // move back to process this item again
                unprocessed.previous();
            } catch (Exception e) {
                if (token.isEmpty()) {
                    LOG.error("Log processing failed", e);
                } else {
                    LOG.error("Log processing failed. Log created but not added to the list!", e);
                }
                importStatus.addError("Log processing failed", e);
                break;
            } finally {
                if (tempFile != null) {
                    tempFile.delete();
                }
            }
        }
        // end else with no condition
        importStatus.finishEntry();
    }
    // end main while loop
    if (dataWasAdded) {
        webhooks.run();
    }
    importStatus.finishList();
    // update log.json
    try {
        logListStore.updateData(Function.identity());
    } catch (IOException e) {
        LOG.error("Updating the log failed", e);
        importStatus.addError("Updating log failed", e);
    }
    return importStatus;
}

Example 2 with RdfPatchSerializer

use of nl.knaw.huygens.timbuctoo.v5.rdfio.RdfPatchSerializer in project timbuctoo by HuygensING.

the class JsonProvenanceToRdfPatch method sendQuads.

@Override
public void sendQuads(RdfPatchSerializer saver, Consumer<String> importStatus, DataSet dataSet) throws LogStorageFailedException {
    for (JsonNode revision : activity.get(PROV_GENERATES)) {
        final String entityUri = revision.get(PROV_SPECIALIZATION_OF).get(0).get("@id").asText();
        final String revisionUri = revision.get("@id").asText();
        String wasRevisionOf = null;
        if (revision.get(PROV_REVISION_OF) != null) {
            wasRevisionOf = revision.get(PROV_REVISION_OF).get(0).get("@id").asText();
        }
        generateRevisionInfo(saver, revisionUri, entityUri, wasRevisionOf);
        generatePatch(saver, revision.get(TIM_ADDITIONS), entityUri, true);
        for (CursorQuad quad : toReplace.getOrDefault(revisionUri, new ArrayList<>())) {
            saver.delQuad(quad.getSubject(), quad.getPredicate(), quad.getObject(), quad.getValuetype().orElse(null), quad.getLanguage().orElse(null), null);
        }
        generatePatch(saver, revision.get(TIM_REPLACEMENTS), entityUri, true);
        generatePatch(saver, revision.get(TIM_DELETIONS), entityUri, false);
    }
    try {
        final HashMap map = OBJECT_MAPPER.treeToValue(activity, HashMap.class);
        final RDFDataset dataset = (RDFDataset) JsonLdProcessor.toRDF(map);
        for (String graphName : dataset.graphNames()) {
            for (RDFDataset.Quad quad : dataset.getQuads(graphName)) {
                saver.onQuad(quad.getSubject().getValue(), quad.getPredicate().getValue(), quad.getObject().getValue(), quad.getObject().isLiteral() ? quad.getObject().getDatatype() : null, quad.getObject().getLanguage(), quad.getGraph() == null ? null : quad.getGraph().getValue());
            }
        }
    } catch (JsonProcessingException | JsonLdError e) {
        throw new LogStorageFailedException(e);
    }
}

Example 3 with RdfPatchSerializer

use of nl.knaw.huygens.timbuctoo.v5.rdfio.RdfPatchSerializer in project timbuctoo by HuygensING.

the class PredicateMutationRdfPatcher method sendQuads.

@Override
public void sendQuads(RdfPatchSerializer saver, Consumer<String> importStatusConsumer, DataSet dataSet) throws LogStorageFailedException {
    final QuadStore quadStore = dataSet.getQuadStore();
    final Map<String, String> foundSubjects = new HashMap<>();
    for (Map.Entry<UUID, PredicateMutation.SubjectFinder> entry : mutation.getSubjectFinders().entrySet()) {
        foundSubjects.put(entry.getKey().toString(), entry.getValue().getSubject(quadStore));
    }
    for (CursorQuad newValue : mutation.getFullRetractions()) {
        final String subject = foundSubjects.getOrDefault(newValue.getSubject(), newValue.getSubject());
        final String predicate = newValue.getPredicate();
        final Direction direction = newValue.getDirection();
        try (Stream<CursorQuad> quads = quadStore.getQuads(subject, predicate, direction, "")) {
            for (CursorQuad oldValue : (Iterable<CursorQuad>) quads::iterator) {
                saver.delQuad(oldValue.getSubject(), oldValue.getPredicate(), oldValue.getObject(), oldValue.getValuetype().orElse(null), oldValue.getLanguage().orElse(null), null);
            }
        }
    }
    for (CursorQuad oldValue : mutation.getRetractions()) {
        saver.delQuad(foundSubjects.getOrDefault(oldValue.getSubject(), oldValue.getSubject()), oldValue.getPredicate(), foundSubjects.getOrDefault(oldValue.getObject(), oldValue.getObject()), oldValue.getValuetype().orElse(null), oldValue.getLanguage().orElse(null), null);
    }
    for (CursorQuad newValue : mutation.getAdditions()) {
        if (newValue.getObject() != null) {
            saver.onQuad(foundSubjects.getOrDefault(newValue.getSubject(), newValue.getSubject()), newValue.getPredicate(), foundSubjects.getOrDefault(newValue.getObject(), newValue.getObject()), newValue.getValuetype().orElse(null), newValue.getLanguage().orElse(null), null);
        }
    }
}