Search in sources :

Example 1 with Tuple

use of com.evolvedbinary.j8fu.tuple.Tuple.Tuple in project exist by eXist-db.

the class BlobStoreImpl method add.

@Override
public Tuple2<BlobId, Long> add(final Txn transaction, final InputStream is) throws IOException {
    if (state.get() != State.OPEN) {
        throw new IOException("Blob Store is not open!");
    }
    // stage the BLOB file
    final Tuple3<Path, Long, MessageDigest> staged = stage(is);
    final BlobVacuum.RequestDeleteStagedBlobFile requestDeleteStagedBlobFile = new BlobVacuum.RequestDeleteStagedBlobFile(stagingDir, staged._1.getFileName().toString());
    // register a callback to cleanup the staged BLOB file ONLY after commit+checkpoint
    final JournalManager journalManager = database.getJournalManager().orElse(null);
    if (journalManager != null) {
        final DeleteStagedBlobFile cleanupStagedBlob = new DeleteStagedBlobFile(vacuumQueue, requestDeleteStagedBlobFile);
        journalManager.listen(cleanupStagedBlob);
        transaction.registerListener(cleanupStagedBlob);
    }
    final BlobId blobId = new BlobId(staged._3.getValue());
    // if the blob entry does not exist, we exclusively compute it as STAGED.
    BlobReference blobReference = references.computeIfAbsent(blobId, k -> new BlobReference(STAGED));
    try {
        while (true) {
            if (blobReference.count.compareAndSet(STAGED, PROMOTING)) {
                // write journal entries to the WAL
                if (journalManager != null) {
                    try {
                        journalManager.journal(new StoreBlobFileLoggable(transaction.getId(), blobId, staged._1.getFileName().toString()));
                        journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, 0, 1));
                        // force WAL entries to disk!
                        journalManager.flush(true, true);
                    } catch (final JournalException e) {
                        references.remove(blobId);
                        throw new IOException(e);
                    }
                }
                // promote the staged blob
                promote(staged);
                if (journalManager == null) {
                    // no journal (or recovery)... so go ahead and schedule cleanup of the staged blob file
                    enqueueVacuum(vacuumQueue, requestDeleteStagedBlobFile);
                }
                // schedule disk persist of the new value
                persistQueue.put(Tuple(blobId, blobReference, 1));
                // update memory with the new value
                blobReference.count.set(1);
                // done!
                return Tuple(blobId, staged._2);
            }
            final int count = blobReference.count.get();
            // guard against a concurrent #add or #remove
            if (count == PROMOTING || count == UPDATING_COUNT) {
                // spin whilst another thread promotes the blob, or updates the reference count
                // sleep a small time to save CPU
                Thread.sleep(10);
                continue;
            }
            // i.e. wait for the deletion of the blob to complete, and then we can add the blob again
            if (count == DELETING) {
                blobReference = references.computeIfAbsent(blobId, k -> new BlobReference(STAGED));
                // loop again
                continue;
            }
            // only increment the blob reference if the blob is active!
            if (count >= 0 && blobReference.count.compareAndSet(count, UPDATING_COUNT)) {
                // NOTE: we are the only thread that can be in this branch for the blobId
                final int newCount = count + 1;
                // write journal entries to the WAL
                if (journalManager != null) {
                    try {
                        journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, count, newCount));
                        // force WAL entries to disk!
                        journalManager.flush(true, true);
                    } catch (final JournalException e) {
                        // restore the state of the blobReference first!
                        blobReference.count.set(count);
                        throw new IOException(e);
                    }
                }
                // persist the new value
                persistQueue.put(Tuple(blobId, blobReference, newCount));
                // update memory with the new value, and release other spinning threads
                blobReference.count.set(newCount);
                // done!
                return Tuple(blobId, staged._2);
            }
        }
    } catch (final InterruptedException e) {
        // thrown by persistQueue.put or Thread.sleep
        Thread.currentThread().interrupt();
        throw new IOException(e);
    }
}
Also used : Path(java.nio.file.Path) Tuple3(com.evolvedbinary.j8fu.tuple.Tuple3) Tuple2(com.evolvedbinary.j8fu.tuple.Tuple2) Txn(org.exist.storage.txn.Txn) JournalException(org.exist.storage.journal.JournalException) RawDataBackup(org.exist.backup.RawDataBackup) ByteBuffer(java.nio.ByteBuffer) FileUtils(org.exist.util.FileUtils) Tuple(com.evolvedbinary.j8fu.tuple.Tuple.Tuple) ThreadUtils.nameInstanceThread(org.exist.util.ThreadUtils.nameInstanceThread) DigestInputStream(org.exist.util.crypto.digest.DigestInputStream) StreamableDigest(org.exist.util.crypto.digest.StreamableDigest) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DigestType(org.exist.util.crypto.digest.DigestType) LOG_UPDATE_BLOB_REF_COUNT(org.exist.storage.blob.BlobLoggable.LOG_UPDATE_BLOB_REF_COUNT) Path(java.nio.file.Path) java.util.concurrent(java.util.concurrent) StandardOpenOption(java.nio.file.StandardOpenOption) FileNotFoundException(java.io.FileNotFoundException) SeekableByteChannel(java.nio.channels.SeekableByteChannel) Logger(org.apache.logging.log4j.Logger) TxnListener(org.exist.storage.txn.TxnListener) LogEntryTypes(org.exist.storage.journal.LogEntryTypes) java.util(java.util) Try(com.evolvedbinary.j8fu.Try) ThreadSafe(net.jcip.annotations.ThreadSafe) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) FilterInputStream(java.io.FilterInputStream) LOG_STORE_BLOB_FILE(org.exist.storage.blob.BlobLoggable.LOG_STORE_BLOB_FILE) ThreadUtils.newInstanceSubThreadGroup(org.exist.util.ThreadUtils.newInstanceSubThreadGroup) LogException(org.exist.storage.journal.LogException) REPLACE_EXISTING(java.nio.file.StandardCopyOption.REPLACE_EXISTING) Nullable(javax.annotation.Nullable) OutputStream(java.io.OutputStream) Database(org.exist.Database) ATOMIC_MOVE(java.nio.file.StandardCopyOption.ATOMIC_MOVE) CountingInputStream(org.apache.commons.io.input.CountingInputStream) Files(java.nio.file.Files) HexEncoder.bytesToHex(org.exist.util.HexEncoder.bytesToHex) JournalManager(org.exist.storage.journal.JournalManager) IOException(java.io.IOException) BlobReference(org.exist.storage.blob.BlobStoreImpl.BlobReference) UUIDGenerator(org.exist.util.UUIDGenerator) MessageDigest(org.exist.util.crypto.digest.MessageDigest) FileUtils.fileName(org.exist.util.FileUtils.fileName) TaggedTryUnchecked(com.evolvedbinary.j8fu.Try.TaggedTryUnchecked) LogManager(org.apache.logging.log4j.LogManager) InputStream(java.io.InputStream) BlobReference(org.exist.storage.blob.BlobStoreImpl.BlobReference) JournalException(org.exist.storage.journal.JournalException) JournalManager(org.exist.storage.journal.JournalManager) IOException(java.io.IOException) MessageDigest(org.exist.util.crypto.digest.MessageDigest)

Aggregations

Try (com.evolvedbinary.j8fu.Try)1 TaggedTryUnchecked (com.evolvedbinary.j8fu.Try.TaggedTryUnchecked)1 Tuple (com.evolvedbinary.j8fu.tuple.Tuple.Tuple)1 Tuple2 (com.evolvedbinary.j8fu.tuple.Tuple2)1 Tuple3 (com.evolvedbinary.j8fu.tuple.Tuple3)1 FileNotFoundException (java.io.FileNotFoundException)1 FilterInputStream (java.io.FilterInputStream)1 IOException (java.io.IOException)1 InputStream (java.io.InputStream)1 OutputStream (java.io.OutputStream)1 ByteBuffer (java.nio.ByteBuffer)1 SeekableByteChannel (java.nio.channels.SeekableByteChannel)1 Files (java.nio.file.Files)1 Path (java.nio.file.Path)1 ATOMIC_MOVE (java.nio.file.StandardCopyOption.ATOMIC_MOVE)1 REPLACE_EXISTING (java.nio.file.StandardCopyOption.REPLACE_EXISTING)1 StandardOpenOption (java.nio.file.StandardOpenOption)1 java.util (java.util)1 java.util.concurrent (java.util.concurrent)1 AtomicBoolean (java.util.concurrent.atomic.AtomicBoolean)1