Examples with BlobReference org.exist.storage.blob.BlobStoreImpl.BlobReference used on opensource projects

Example 1 with BlobReference

use of org.exist.storage.blob.BlobStoreImpl.BlobReference in project exist by eXist-db.

the class BlobStoreImpl method compactPersistentReferences.

/**
 * Compacts an existing Blob Store file.
 *
 * Reads the existing Blob Store file and copies non zero reference
 * entries to a new Blob Store file. We call this compaction.
 * Once complete, the existing file is replaced with the new file.
 *
 * @param persistentFile an existing persistentFile to compact.
 *
 * @return An in-memory representation of the compacted Blob Store
 *
 * @throws IOException if an error occurs during compaction.
 */
private ConcurrentMap<BlobId, BlobReference> compactPersistentReferences(final ByteBuffer buffer, final Path persistentFile) throws IOException {
    final ConcurrentMap<BlobId, BlobReference> compactReferences = new ConcurrentHashMap<>();
    final Path compactPersistentFile = persistentFile.getParent().resolve(persistentFile.getFileName() + ".new." + System.currentTimeMillis());
    // tracks the BlobIds of Blob Files which have been orphaned
    final Set<BlobId> orphanedBlobFileIds = new HashSet<>();
    try (final SeekableByteChannel channel = Files.newByteChannel(persistentFile, READ)) {
        validateFileHeader(buffer, persistentFile, channel);
        buffer.clear();
        try (final SeekableByteChannel compactChannel = Files.newByteChannel(compactPersistentFile, CREATE_NEW, APPEND)) {
            writeFileHeader(buffer, compactChannel);
            buffer.clear();
            while (channel.read(buffer) > -1) {
                final byte[] id = new byte[digestType.getDigestLengthBytes()];
                buffer.flip();
                buffer.get(id);
                final BlobId blobId = new BlobId(id);
                final int count = buffer.getInt();
                if (count == 0) {
                    orphanedBlobFileIds.add(blobId);
                } else {
                    orphanedBlobFileIds.remove(blobId);
                    compactReferences.put(blobId, new BlobReference(count, compactChannel.position()));
                    buffer.flip();
                    compactChannel.write(buffer);
                }
                buffer.clear();
            }
        }
    }
    // cleanup any orphaned Blob files
    for (final BlobId orphanedBlobFileId : orphanedBlobFileIds) {
        deleteBlob(blobDir, orphanedBlobFileId, false);
    }
    // replace the persistent file with the new compact persistent file
    Files.move(compactPersistentFile, persistentFile, ATOMIC_MOVE, REPLACE_EXISTING);
    return compactReferences;
}

Example 2 with BlobReference

use of org.exist.storage.blob.BlobStoreImpl.BlobReference in project exist by eXist-db.

the class BlobStoreImpl method add.

@Override
public Tuple2<BlobId, Long> add(final Txn transaction, final InputStream is) throws IOException {
    if (state.get() != State.OPEN) {
        throw new IOException("Blob Store is not open!");
    }
    // stage the BLOB file
    final Tuple3<Path, Long, MessageDigest> staged = stage(is);
    final BlobVacuum.RequestDeleteStagedBlobFile requestDeleteStagedBlobFile = new BlobVacuum.RequestDeleteStagedBlobFile(stagingDir, staged._1.getFileName().toString());
    // register a callback to cleanup the staged BLOB file ONLY after commit+checkpoint
    final JournalManager journalManager = database.getJournalManager().orElse(null);
    if (journalManager != null) {
        final DeleteStagedBlobFile cleanupStagedBlob = new DeleteStagedBlobFile(vacuumQueue, requestDeleteStagedBlobFile);
        journalManager.listen(cleanupStagedBlob);
        transaction.registerListener(cleanupStagedBlob);
    }
    final BlobId blobId = new BlobId(staged._3.getValue());
    // if the blob entry does not exist, we exclusively compute it as STAGED.
    BlobReference blobReference = references.computeIfAbsent(blobId, k -> new BlobReference(STAGED));
    try {
        while (true) {
            if (blobReference.count.compareAndSet(STAGED, PROMOTING)) {
                // write journal entries to the WAL
                if (journalManager != null) {
                    try {
                        journalManager.journal(new StoreBlobFileLoggable(transaction.getId(), blobId, staged._1.getFileName().toString()));
                        journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, 0, 1));
                        // force WAL entries to disk!
                        journalManager.flush(true, true);
                    } catch (final JournalException e) {
                        references.remove(blobId);
                        throw new IOException(e);
                    }
                }
                // promote the staged blob
                promote(staged);
                if (journalManager == null) {
                    // no journal (or recovery)... so go ahead and schedule cleanup of the staged blob file
                    enqueueVacuum(vacuumQueue, requestDeleteStagedBlobFile);
                }
                // schedule disk persist of the new value
                persistQueue.put(Tuple(blobId, blobReference, 1));
                // update memory with the new value
                blobReference.count.set(1);
                // done!
                return Tuple(blobId, staged._2);
            }
            final int count = blobReference.count.get();
            // guard against a concurrent #add or #remove
            if (count == PROMOTING || count == UPDATING_COUNT) {
                // spin whilst another thread promotes the blob, or updates the reference count
                // sleep a small time to save CPU
                Thread.sleep(10);
                continue;
            }
            // i.e. wait for the deletion of the blob to complete, and then we can add the blob again
            if (count == DELETING) {
                blobReference = references.computeIfAbsent(blobId, k -> new BlobReference(STAGED));
                // loop again
                continue;
            }
            // only increment the blob reference if the blob is active!
            if (count >= 0 && blobReference.count.compareAndSet(count, UPDATING_COUNT)) {
                // NOTE: we are the only thread that can be in this branch for the blobId
                final int newCount = count + 1;
                // write journal entries to the WAL
                if (journalManager != null) {
                    try {
                        journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, count, newCount));
                        // force WAL entries to disk!
                        journalManager.flush(true, true);
                    } catch (final JournalException e) {
                        // restore the state of the blobReference first!
                        blobReference.count.set(count);
                        throw new IOException(e);
                    }
                }
                // persist the new value
                persistQueue.put(Tuple(blobId, blobReference, newCount));
                // update memory with the new value, and release other spinning threads
                blobReference.count.set(newCount);
                // done!
                return Tuple(blobId, staged._2);
            }
        }
    } catch (final InterruptedException e) {
        // thrown by persistQueue.put or Thread.sleep
        Thread.currentThread().interrupt();
        throw new IOException(e);
    }
}

Example 3 with BlobReference

use of org.exist.storage.blob.BlobStoreImpl.BlobReference in project exist by eXist-db.

the class BlobStoreImpl method readLeaseBlobFile.

/**
 * Lease a Blob file for reading from the Blob Store.
 *
 * @param transaction the current database transaction.
 * @param blobId the identifier of the blob to lease the blob file from.
 *
 * @return the blob file lease, or null if the blob does not exist in the Blob Store
 *
 * @throws IOException if an error occurs whilst retrieving the BLOB file.
 */
private BlobFileLease readLeaseBlobFile(final Txn transaction, final BlobId blobId) throws IOException {
    if (state.get() != State.OPEN) {
        throw new IOException("Blob Store is not open!");
    }
    final BlobReference blobReference = references.get(blobId);
    if (blobReference == null) {
        return null;
    }
    try {
        while (true) {
            final int count = blobReference.count.get();
            if (count == 0) {
                // can't return something with has zero references
                return null;
            }
            // guard against a concurrent vacuum operation
            if (count == DELETING) {
                // can't return something with has zero references (because it is being deleted)
                return null;
            }
            // guard against a concurrent #add doing staging
            if (count == STAGED || count == PROMOTING) {
                // spin whilst another thread promotes the blob
                // sleep a small time to save CPU
                Thread.sleep(10);
                continue;
            }
            // read a blob which has references
            if (count > 0) {
                // we are reading
                blobReference.readers.incrementAndGet();
                // get the blob
                final Path blobFile = blobDir.resolve(bytesToHex(blobId.getId()));
                return new BlobFileLease(blobFile, blobReference.readers::decrementAndGet);
            }
        }
    } catch (final InterruptedException e) {
        // only thrown by Thread.sleep above
        Thread.currentThread().interrupt();
        throw new IOException(e);
    }
}

Example 4 with BlobReference

use of org.exist.storage.blob.BlobStoreImpl.BlobReference in project exist by eXist-db.

the class BlobStoreImpl method copy.

@Override
public BlobId copy(final Txn transaction, final BlobId blobId) throws IOException {
    if (state.get() != State.OPEN) {
        throw new IOException("Blob Store is not open!");
    }
    final BlobReference blobReference = references.get(blobId);
    if (blobReference == null) {
        return null;
    }
    // NOTE: that copy is simply an increment of the reference count!
    try {
        while (true) {
            final int count = blobReference.count.get();
            // guard against a concurrent #add or #remove
            if (count == STAGED || count == PROMOTING || count == UPDATING_COUNT) {
                // spin whilst another thread promotes the blob, or updates the reference count
                // sleep a small time to save CPU
                Thread.sleep(10);
                continue;
            }
            // guard against a concurrent vacuum operation
            if (count == DELETING) {
                return null;
            }
            // only increment the blob reference if the blob is active!
            if (count >= 0 && blobReference.count.compareAndSet(count, UPDATING_COUNT)) {
                // NOTE: we are the only thread that can be in this branch for the blobId
                final int newCount = count + 1;
                // write journal entries to the WAL
                final JournalManager journalManager = database.getJournalManager().orElse(null);
                if (journalManager != null) {
                    try {
                        journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, count, newCount));
                        // force WAL entries to disk!
                        journalManager.flush(true, true);
                    } catch (final JournalException e) {
                        // restore the state of the blobReference first!
                        blobReference.count.set(count);
                        throw new IOException(e);
                    }
                }
                // persist the new value
                persistQueue.put(Tuple(blobId, blobReference, newCount));
                // update memory with the new value, and release other spinning threads
                blobReference.count.set(newCount);
                // done!
                return blobId;
            }
        }
    } catch (final InterruptedException e) {
        // thrown by persistQueue.put or Thread.sleep
        Thread.currentThread().interrupt();
        throw new IOException(e);
    }
}

Example 5 with BlobReference

use of org.exist.storage.blob.BlobStoreImpl.BlobReference in project exist by eXist-db.

the class BlobStoreImpl method remove.

@Override
public void remove(final Txn transaction, final BlobId blobId) throws IOException {
    if (state.get() != State.OPEN) {
        throw new IOException("Blob Store is not open!");
    }
    final BlobReference blobReference = references.get(blobId);
    if (blobReference == null) {
        return;
    }
    try {
        while (true) {
            final int count = blobReference.count.get();
            if (count == 0) {
                // can't remove something which has zero references
                return;
            }
            // guard against a concurrent vacuum operation
            if (count == DELETING) {
                // can't remove something which has zero references (because it is being deleted)
                return;
            }
            // guard against a concurrent #add or #remove
            if (count == STAGED || count == PROMOTING || count == UPDATING_COUNT) {
                // spin whilst another thread promotes the blob or updates the reference count
                // sleep a small time to save CPU
                Thread.sleep(10);
                continue;
            }
            // only decrement the blob reference if the blob has more than zero references
            if (count > 0 && blobReference.count.compareAndSet(count, UPDATING_COUNT)) {
                // NOTE: we are the only thread that can be in this branch for the blobId
                final int newCount = count - 1;
                // write journal entries to the WAL
                final JournalManager journalManager = database.getJournalManager().orElse(null);
                if (journalManager != null) {
                    try {
                        journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, count, newCount));
                        // force WAL entries to disk!
                        journalManager.flush(true, true);
                    } catch (final JournalException e) {
                        // restore the state of the blobReference first!
                        blobReference.count.set(count);
                        throw new IOException(e);
                    }
                }
                // schedule disk persist of the new value
                persistQueue.put(Tuple(blobId, blobReference, newCount));
                if (newCount == 0) {
                    // schedule blob file for vacuum.
                    final BlobVacuum.RequestDeleteBlobFile requestDeleteBlobFile = new BlobVacuum.RequestDeleteBlobFile(references, blobDir, blobId, blobReference);
                    if (journalManager != null) {
                        // register a callback to schedule the BLOB file for vacuum ONLY after commit+checkpoint
                        final ScheduleDeleteBlobFile scheduleDeleteBlobFile = new ScheduleDeleteBlobFile(vacuumQueue, requestDeleteBlobFile);
                        journalManager.listen(scheduleDeleteBlobFile);
                        transaction.registerListener(scheduleDeleteBlobFile);
                    } else {
                        // no journal (or recovery)... so go ahead and schedule
                        enqueueVacuum(vacuumQueue, requestDeleteBlobFile);
                    }
                }
                // update memory with the new value, and release other spinning threads
                blobReference.count.set(newCount);
                // done!
                return;
            }
        }
    } catch (final InterruptedException e) {
        // thrown by persistQueue.put or Thread.sleep
        Thread.currentThread().interrupt();
        throw new IOException(e);
    }
}