Examples with BlobStore org.exist.storage.blob.BlobStore used on opensource projects

Example 1 with BlobStore

use of org.exist.storage.blob.BlobStore in project exist by eXist-db.

the class NativeBroker method storeBinaryResource.

@Override
public void storeBinaryResource(final Txn transaction, final BinaryDocument blob, final InputStream is) throws IOException {
    final BlobStore blobStore = pool.getBlobStore();
    final Tuple2<BlobId, Long> blobIdLen = blobStore.add(transaction, is);
    blob.setBlobId(blobIdLen._1);
    blob.setContentLength(blobIdLen._2);
}

Example 2 with BlobStore

use of org.exist.storage.blob.BlobStore in project exist by eXist-db.

the class NativeBroker method copyBinaryResource.

private void copyBinaryResource(final Txn transaction, final BinaryDocument srcDoc, final BinaryDocument dstDoc) throws IOException {
    final BlobStore blobStore = pool.getBlobStore();
    final BlobId dstBlobId = blobStore.copy(transaction, srcDoc.getBlobId());
    dstDoc.setBlobId(dstBlobId);
    dstDoc.setContentLength(srcDoc.getContentLength());
}

Example 3 with BlobStore

use of org.exist.storage.blob.BlobStore in project exist by eXist-db.

the class MetadataFunctions method extractMetadataFromLocalResource.

private Sequence extractMetadataFromLocalResource(final XmldbURI docUri) throws XPathException {
    try (final LockedDocument lockedDoc = context.getBroker().getXMLResource(docUri, LockMode.READ_LOCK)) {
        if (lockedDoc != null && lockedDoc.getDocument() instanceof BinaryDocument) {
            final BinaryDocument binDoc = (BinaryDocument) lockedDoc.getDocument();
            final BrokerPool pool = context.getBroker().getBrokerPool();
            final BlobStore blobStore = pool.getBlobStore();
            try (final Txn transaction = pool.getTransactionManager().beginTransaction()) {
                final Sequence result = blobStore.with(transaction, binDoc.getBlobId(), blobFile -> TaggedTryUnchecked(XPathException.class, () -> exifToolExtract(blobFile))).get();
                transaction.commit();
                return result;
            }
        } else {
            throw new XPathException(this, "The binary document at " + docUri.toString() + " cannot be found.");
        }
    } catch (PermissionDeniedException | IOException | TransactionException e) {
        throw new XPathException(this, "Could not access binary document: " + e.getMessage(), e);
    }
}

Example 4 with BlobStore

use of org.exist.storage.blob.BlobStore in project exist by eXist-db.

the class RecoveryManager method recover.

/**
 * Checks if the database is in a consistent state. If not, start a recovery run.
 *
 * The method scans the last log file and tries to find the last checkpoint
 * record. If the checkpoint record is the last record in the file,
 * the database was closed cleanly and is in a consistent state. If not, a
 * recovery run is started beginning at the last checkpoint found.
 *
 * @throws LogException Reading of journal failed.
 * @return if recover was successful
 */
public boolean recover() throws LogException {
    boolean recoveryRun = false;
    final List<Path> files;
    try (final Stream<Path> fileStream = journalRecovery.getFiles.get()) {
        files = fileStream.collect(Collectors.toList());
    } catch (final IOException ioe) {
        throw new LogException("Unable to find journal files in data dir", ioe);
    }
    // find the last log file in the data directory
    final short lastNum = Journal.findLastFile(files.stream());
    if (-1 < lastNum) {
        // load the last log file
        final Path last = journalRecovery.getFile.apply(lastNum);
        // scan the last log file and record the last checkpoint found
        try (JournalReader reader = new JournalReader(broker, last, lastNum)) {
            // try to read the last log record to see if it is a checkpoint
            boolean checkpointFound = false;
            try {
                final Loggable lastLog = reader.lastEntry();
                if (lastLog != null && lastLog.getLogType() == LogEntryTypes.CHECKPOINT) {
                    final Checkpoint checkpoint = (Checkpoint) lastLog;
                    // record, we compare the LSN stored in it with the current LSN.
                    if (checkpoint.getStoredLsn().equals(checkpoint.getLsn())) {
                        checkpointFound = true;
                        LOG.debug("Database is in clean state. Last checkpoint: {}", checkpoint.getDateString());
                    }
                }
            } catch (final LogException e) {
                LOG.info("Reading last journal log entry failed: {}. Will scan the log...", e.getMessage());
                // if an exception occurs at this point, the journal file is probably incomplete,
                // which indicates a db crash
                checkpointFound = false;
            }
            if (!checkpointFound) {
                LOG.info("Unclean shutdown detected. Scanning journal...");
                broker.getBrokerPool().reportStatus("Unclean shutdown detected. Scanning log...");
                reader.positionFirst();
                final Long2ObjectMap<Loggable> txnsStarted = new Long2ObjectOpenHashMap<>();
                Checkpoint lastCheckpoint = null;
                Lsn lastLsn = Lsn.LSN_INVALID;
                Loggable next;
                try {
                    final ProgressBar progress = new ProgressBar("Scanning journal ", FileUtils.sizeQuietly(last));
                    while ((next = reader.nextEntry()) != null) {
                        // LOG.debug(next.dump());
                        progress.set(next.getLsn().getOffset());
                        if (next.getLogType() == LogEntryTypes.TXN_START) {
                            // new transaction starts: add it to the transactions table
                            txnsStarted.put(next.getTransactionId(), next);
                        } else if (next.getLogType() == LogEntryTypes.TXN_ABORT) {
                            // transaction aborted: remove it from the transactions table
                            txnsStarted.remove(next.getTransactionId());
                        } else if (next.getLogType() == LogEntryTypes.CHECKPOINT) {
                            txnsStarted.clear();
                            lastCheckpoint = (Checkpoint) next;
                        }
                        lastLsn = next.getLsn();
                    }
                } catch (final LogException e) {
                    if (LOG.isDebugEnabled()) {
                        LOG.debug("Caught exception while reading log", e);
                    }
                    LOG.warn("Last readable journal log entry lsn: {}", lastLsn);
                }
                // we need a recovery.
                if ((lastCheckpoint == null || !lastCheckpoint.getLsn().equals(lastLsn)) && txnsStarted.size() > 0) {
                    LOG.info("Dirty transactions: {}", txnsStarted.size());
                    // starting recovery: reposition the log reader to the last checkpoint
                    if (lastCheckpoint == null) {
                        reader.positionFirst();
                    } else {
                        reader.position(lastCheckpoint.getLsn());
                        next = reader.nextEntry();
                    }
                    recoveryRun = true;
                    try {
                        LOG.info("Running recovery...");
                        broker.getBrokerPool().reportStatus("Running recovery...");
                        try (final BlobStore blobStore = broker.getBrokerPool().getBlobStore()) {
                            try {
                                blobStore.openForRecovery();
                            } catch (final FileNotFoundException e) {
                                LOG.warn(e.getMessage(), e);
                            } catch (final IOException e) {
                                throw new LogException("Unable to Open the Blob Store for Recovery: " + e.getMessage(), e);
                            }
                            doRecovery(txnsStarted.size(), last, reader, lastLsn);
                        } catch (final IOException e) {
                            LOG.error("Error whilst closing the Blob Store after recovery: {}", e.getMessage(), e);
                        }
                    } catch (final LogException e) {
                        // if restartOnError == true, we try to bring up the database even if there
                        // are errors. Otherwise, an exception is thrown, which will stop the db initialization
                        broker.getBrokerPool().reportStatus(BrokerPool.SIGNAL_ABORTED);
                        if (restartOnError) {
                            LOG.error("Aborting recovery. eXist-db detected an error during recovery. This may not be fatal. Database will start up, but corruptions are likely.");
                        } else {
                            LOG.error("Aborting recovery. eXist-db detected an error during recovery. This may not be fatal. Please consider running a consistency check via the export tool and create a backup if problems are reported. The db should come up again if you restart it.");
                            throw e;
                        }
                    }
                } else {
                    LOG.info("Database is in clean state. Nothing to recover from the journal.");
                }
            }
        } finally {
            // remove .log files from directory even if recovery failed.
            // Re-applying them on a second start up attempt would definitely damage the db, so we better
            // delete them before user tries to launch again.
            cleanDirectory(files.stream());
            if (recoveryRun) {
                broker.repairPrimary();
                broker.sync(Sync.MAJOR);
            }
        }
    }
    journalRecovery.setCurrentFileNum.accept(lastNum);
    journalRecovery.switchFiles.get();
    return recoveryRun;
}

Example 5 with BlobStore

use of org.exist.storage.blob.BlobStore in project exist by eXist-db.

the class NativeBroker method removeBinaryResource.

@Override
public void removeBinaryResource(final Txn transaction, final BinaryDocument blob) throws PermissionDeniedException, IOException {
    if (isReadOnly()) {
        throw new IOException(DATABASE_IS_READ_ONLY);
    }
    if (LOG.isDebugEnabled()) {
        LOG.debug("removing binary resource {}...", blob.getDocId());
    }
    if (blob.getBlobId() == null) {
        LOG.warn("Trying to delete binary document: {}, but blobId was null", blob.getURI());
        return;
    }
    final BlobStore blobStore = pool.getBlobStore();
    blobStore.remove(transaction, blob.getBlobId());
    // remove the file from the database metadata and indexes
    removeResourceMetadata(transaction, blob);
    getIndexController().setDocument(blob, ReindexMode.REMOVE_BINARY);
    getIndexController().flush();
}