Search in sources :

Example 1 with RdfProcessingFailedException

use of nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException in project timbuctoo by HuygensING.

the class ImportManager method processLogsUntil.

private synchronized ImportStatus processLogsUntil(int maxIndex) {
    importStatus.start(this.getClass().getSimpleName() + ".processLogs", null);
    ListIterator<LogEntry> unprocessed = logListStore.getData().getUnprocessed();
    boolean dataWasAdded = false;
    while (unprocessed.hasNext() && unprocessed.nextIndex() <= maxIndex) {
        int index = unprocessed.nextIndex();
        LogEntry entry = unprocessed.next();
        importStatus.startEntry(entry);
        if (entry.getLogToken().isPresent()) {
            // logToken
            String logToken = entry.getLogToken().get();
            try (CachedLog log = logStorage.getLog(logToken)) {
                final Stopwatch stopwatch = Stopwatch.createStarted();
                for (RdfProcessor processor : subscribedProcessors) {
                    if (processor.getCurrentVersion() <= index) {
                        String msg = "******* " + processor.getClass().getSimpleName() + " Started importing full log...";
                        LOG.info(msg);
                        importStatus.setStatus(msg);
                        RdfParser rdfParser = serializerFactory.makeRdfParser(log);
                        processor.start(index);
                        rdfParser.importRdf(log, entry.getBaseUri(), entry.getDefaultGraph(), processor);
                        processor.commit();
                    }
                }
                long elapsedTime = stopwatch.elapsed(TimeUnit.SECONDS);
                String msg = "Finished importing. Total import took " + elapsedTime + " seconds.";
                LOG.info(msg);
                importStatus.setStatus(msg);
                dataWasAdded = true;
            } catch (Exception e) {
                LOG.error("Processing log failed", e);
                importStatus.addError("Processing log failed", e);
            }
            // Update the log, even after RdfProcessingFailedException | IOException
            try {
                logListStore.updateData(logList -> {
                    logList.markAsProcessed(index);
                    return logList;
                });
            } catch (IOException e) {
                LOG.error("Updating the log failed", e);
                importStatus.addError("Updating log failed", e);
            }
        } else {
            // no logToken
            RdfCreator creator = entry.getRdfCreator().get();
            String token = "";
            MediaType mediaType;
            Optional<Charset> charset;
            File tempFile = null;
            try {
                tempFile = File.createTempFile("log_to_generate", "nq");
                try (OutputStream stream = new GZIPOutputStream(new FileOutputStream(tempFile))) {
                    if (creator instanceof PlainRdfCreator) {
                        try (RdfSerializer serializer = serializerFactory.makeRdfSerializer(stream)) {
                            mediaType = serializer.getMediaType();
                            charset = Optional.of(serializer.getCharset());
                            ((PlainRdfCreator) creator).sendQuads(serializer, dataSet, importStatus::setStatus);
                        } catch (Exception e) {
                            LOG.error("Log generation failed", e);
                            importStatus.addError("Log generation failed", e);
                            break;
                        }
                    } else {
                        try (RdfPatchSerializer srlzr = serializerFactory.makeRdfPatchSerializer(stream, entry.getBaseUri())) {
                            mediaType = srlzr.getMediaType();
                            charset = Optional.of(srlzr.getCharset());
                            ((PatchRdfCreator) creator).sendQuads(srlzr, importStatus::setStatus, dataSet);
                        } catch (Exception e) {
                            LOG.error("Log generation failed", e);
                            importStatus.addError("Log generation failed", e);
                            break;
                        }
                    }
                }
                try (InputStream inputStream = new GZIPInputStream(new FileInputStream(tempFile))) {
                    token = logStorage.saveLog(inputStream, "log_generated_by_" + creator.getClass().getSimpleName(), mediaType, charset);
                }
                LogEntry entryWithLog;
                entryWithLog = LogEntry.addLogToEntry(entry, token);
                unprocessed.set(entryWithLog);
                token = "";
                // move back to process this item again
                unprocessed.previous();
            } catch (Exception e) {
                if (token.isEmpty()) {
                    LOG.error("Log processing failed", e);
                } else {
                    LOG.error("Log processing failed. Log created but not added to the list!", e);
                }
                importStatus.addError("Log processing failed", e);
                break;
            } finally {
                if (tempFile != null) {
                    tempFile.delete();
                }
            }
        }
        // end else with no condition
        importStatus.finishEntry();
    }
    // end main while loop
    if (dataWasAdded) {
        webhooks.run();
    }
    importStatus.finishList();
    // update log.json
    try {
        logListStore.updateData(Function.identity());
    } catch (IOException e) {
        LOG.error("Updating the log failed", e);
        importStatus.addError("Updating log failed", e);
    }
    return importStatus;
}
Also used : OutputStream(java.io.OutputStream) FileOutputStream(java.io.FileOutputStream) GZIPOutputStream(java.util.zip.GZIPOutputStream) Stopwatch(com.google.common.base.Stopwatch) GZIPInputStream(java.util.zip.GZIPInputStream) GZIPOutputStream(java.util.zip.GZIPOutputStream) MediaType(javax.ws.rs.core.MediaType) CachedLog(nl.knaw.huygens.timbuctoo.v5.filestorage.dto.CachedLog) LogEntry(nl.knaw.huygens.timbuctoo.v5.dataset.dto.LogEntry) GZIPInputStream(java.util.zip.GZIPInputStream) FileInputStream(java.io.FileInputStream) InputStream(java.io.InputStream) Charset(java.nio.charset.Charset) IOException(java.io.IOException) LogStorageFailedException(nl.knaw.huygens.timbuctoo.v5.filestorage.exceptions.LogStorageFailedException) IOException(java.io.IOException) FileStorageFailedException(nl.knaw.huygens.timbuctoo.v5.filestorage.exceptions.FileStorageFailedException) FileInputStream(java.io.FileInputStream) RdfSerializer(nl.knaw.huygens.timbuctoo.v5.rdfio.RdfSerializer) RdfPatchSerializer(nl.knaw.huygens.timbuctoo.v5.rdfio.RdfPatchSerializer) FileOutputStream(java.io.FileOutputStream) RdfCreator(nl.knaw.huygens.timbuctoo.v5.dataset.dto.RdfCreator) CachedFile(nl.knaw.huygens.timbuctoo.v5.filestorage.dto.CachedFile) File(java.io.File) RdfParser(nl.knaw.huygens.timbuctoo.v5.rdfio.RdfParser)

Example 2 with RdfProcessingFailedException

use of nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException in project timbuctoo by HuygensING.

the class RdfDescriptionSaver method delValue.

@Override
public void delValue(String subject, String predicate, String value, String dataType, String graph) throws RdfProcessingFailedException {
    try {
        if (Objects.equals(subject, baseUri)) {
            ValueFactory vf = SimpleValueFactory.getInstance();
            model.remove(vf.createIRI(subject), vf.createIRI(predicate), vf.createLiteral(value));
        }
    } catch (Exception e) {
        throw new RdfProcessingFailedException(e);
    }
}
Also used : ValueFactory(org.eclipse.rdf4j.model.ValueFactory) SimpleValueFactory(org.eclipse.rdf4j.model.impl.SimpleValueFactory) RdfProcessingFailedException(nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException) RdfProcessingFailedException(nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException) IOException(java.io.IOException) ParserConfigurationException(javax.xml.parsers.ParserConfigurationException) SAXException(org.xml.sax.SAXException)

Example 3 with RdfProcessingFailedException

use of nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException in project timbuctoo by HuygensING.

the class RdfDescriptionSaver method addRelation.

@Override
public void addRelation(String subject, String predicate, String object, String graph) throws RdfProcessingFailedException {
    try {
        if (Objects.equals(subject, baseUri) && isDescriptionPredicate(predicate)) {
            ValueFactory vf = SimpleValueFactory.getInstance();
            model.add(vf.createIRI(subject), vf.createIRI(predicate), vf.createIRI(object));
        }
    } catch (Exception e) {
        throw new RdfProcessingFailedException(e);
    }
}
Also used : ValueFactory(org.eclipse.rdf4j.model.ValueFactory) SimpleValueFactory(org.eclipse.rdf4j.model.impl.SimpleValueFactory) RdfProcessingFailedException(nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException) RdfProcessingFailedException(nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException) IOException(java.io.IOException) ParserConfigurationException(javax.xml.parsers.ParserConfigurationException) SAXException(org.xml.sax.SAXException)

Example 4 with RdfProcessingFailedException

use of nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException in project timbuctoo by HuygensING.

the class BdbRmlDataSourceStore method onChangedSubject.

@Override
public void onChangedSubject(String subject, ChangeFetcher changeFetcher) throws RdfProcessingFailedException {
    try {
        boolean[] wasCollection = new boolean[] { false };
        StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, TIM_HAS_ROW, Direction.OUT, true, false, true), quad -> {
            wasCollection[0] = true;
            if (quad.getChangeType() == ChangeType.ASSERTED) {
                bdbWrapper.put(subject, quad.getObject());
            }
        });
        if (!wasCollection[0]) {
            try (Stream<CursorQuad> quads = changeFetcher.getPredicates(subject, TIM_HAS_ROW, Direction.IN, true, true, true)) {
                Optional<CursorQuad> isRawRow = quads.findFirst();
                if (isRawRow.isPresent()) {
                    final String collectionUri = isRawRow.get().getObject();
                    Map<String, Property> predicatesToStore = collectionProperties.computeIfAbsent(collectionUri, collection -> getPropertyNames(changeFetcher, collection));
                    switch(isRawRow.get().getChangeType()) {
                        case ASSERTED:
                            // add all unchanged and new predicates
                            StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, false, true, true), pred -> {
                                final Property property = predicatesToStore.get(pred.getPredicate());
                                if (property != null) {
                                    bdbWrapper.put(subject, property.newName + "\n" + pred.getObject());
                                }
                            });
                            break;
                        case RETRACTED:
                            // remove all unchanged and removed predicates
                            StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, true, true, false), pred -> {
                                final Property property = predicatesToStore.get(pred.getPredicate());
                                if (property != null) {
                                    bdbWrapper.delete(subject, property.oldName + "\n" + pred.getObject());
                                }
                            });
                            break;
                        case UNCHANGED:
                            // add all added predicates, remove all removed predicates
                            StreamIterator.iterateAndCloseOrThrow(changeFetcher.getPredicates(subject, true, false, true), pred -> {
                                final Property property = predicatesToStore.get(pred.getPredicate());
                                if (property != null) {
                                    if (pred.getChangeType() == ChangeType.RETRACTED) {
                                        bdbWrapper.delete(subject, property.oldName + "\n" + pred.getObject());
                                    } else {
                                        bdbWrapper.put(subject, property.oldName + "\n" + pred.getObject());
                                    }
                                }
                            });
                            break;
                        default:
                            throw new RuntimeException("Should not happen");
                    }
                }
            }
        }
    } catch (DatabaseWriteException e) {
        throw new RdfProcessingFailedException(e);
    }
}
Also used : CursorQuad(nl.knaw.huygens.timbuctoo.v5.datastores.quadstore.dto.CursorQuad) DatabaseWriteException(nl.knaw.huygens.timbuctoo.v5.berkeleydb.exceptions.DatabaseWriteException) RdfProcessingFailedException(nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException)

Example 5 with RdfProcessingFailedException

use of nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException in project timbuctoo by HuygensING.

the class RdfDescriptionSaver method delRelation.

@Override
public void delRelation(String subject, String predicate, String object, String graph) throws RdfProcessingFailedException {
    try {
        if (Objects.equals(subject, baseUri)) {
            ValueFactory vf = SimpleValueFactory.getInstance();
            model.remove(vf.createIRI(subject), vf.createIRI(predicate), vf.createIRI(object));
        }
    } catch (Exception e) {
        throw new RdfProcessingFailedException(e);
    }
}
Also used : ValueFactory(org.eclipse.rdf4j.model.ValueFactory) SimpleValueFactory(org.eclipse.rdf4j.model.impl.SimpleValueFactory) RdfProcessingFailedException(nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException) RdfProcessingFailedException(nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException) IOException(java.io.IOException) ParserConfigurationException(javax.xml.parsers.ParserConfigurationException) SAXException(org.xml.sax.SAXException)

Aggregations

RdfProcessingFailedException (nl.knaw.huygens.timbuctoo.v5.dataset.exceptions.RdfProcessingFailedException)11 IOException (java.io.IOException)7 ParserConfigurationException (javax.xml.parsers.ParserConfigurationException)4 DatabaseWriteException (nl.knaw.huygens.timbuctoo.v5.berkeleydb.exceptions.DatabaseWriteException)4 ValueFactory (org.eclipse.rdf4j.model.ValueFactory)4 SimpleValueFactory (org.eclipse.rdf4j.model.impl.SimpleValueFactory)4 SAXException (org.xml.sax.SAXException)4 RDFHandlerException (org.eclipse.rdf4j.rio.RDFHandlerException)2 JsonProcessingException (com.fasterxml.jackson.core.JsonProcessingException)1 Stopwatch (com.google.common.base.Stopwatch)1 File (java.io.File)1 FileInputStream (java.io.FileInputStream)1 FileOutputStream (java.io.FileOutputStream)1 FileWriter (java.io.FileWriter)1 InputStream (java.io.InputStream)1 OutputStream (java.io.OutputStream)1 Charset (java.nio.charset.Charset)1 GZIPInputStream (java.util.zip.GZIPInputStream)1 GZIPOutputStream (java.util.zip.GZIPOutputStream)1 MediaType (javax.ws.rs.core.MediaType)1