Search in sources :

Example 1 with Tuple3

use of com.evolvedbinary.j8fu.tuple.Tuple3 in project exist by eXist-db.

the class BlobStoreImpl method add.

@Override
public Tuple2<BlobId, Long> add(final Txn transaction, final InputStream is) throws IOException {
    if (state.get() != State.OPEN) {
        throw new IOException("Blob Store is not open!");
    }
    // stage the BLOB file
    final Tuple3<Path, Long, MessageDigest> staged = stage(is);
    final BlobVacuum.RequestDeleteStagedBlobFile requestDeleteStagedBlobFile = new BlobVacuum.RequestDeleteStagedBlobFile(stagingDir, staged._1.getFileName().toString());
    // register a callback to cleanup the staged BLOB file ONLY after commit+checkpoint
    final JournalManager journalManager = database.getJournalManager().orElse(null);
    if (journalManager != null) {
        final DeleteStagedBlobFile cleanupStagedBlob = new DeleteStagedBlobFile(vacuumQueue, requestDeleteStagedBlobFile);
        journalManager.listen(cleanupStagedBlob);
        transaction.registerListener(cleanupStagedBlob);
    }
    final BlobId blobId = new BlobId(staged._3.getValue());
    // if the blob entry does not exist, we exclusively compute it as STAGED.
    BlobReference blobReference = references.computeIfAbsent(blobId, k -> new BlobReference(STAGED));
    try {
        while (true) {
            if (blobReference.count.compareAndSet(STAGED, PROMOTING)) {
                // write journal entries to the WAL
                if (journalManager != null) {
                    try {
                        journalManager.journal(new StoreBlobFileLoggable(transaction.getId(), blobId, staged._1.getFileName().toString()));
                        journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, 0, 1));
                        // force WAL entries to disk!
                        journalManager.flush(true, true);
                    } catch (final JournalException e) {
                        references.remove(blobId);
                        throw new IOException(e);
                    }
                }
                // promote the staged blob
                promote(staged);
                if (journalManager == null) {
                    // no journal (or recovery)... so go ahead and schedule cleanup of the staged blob file
                    enqueueVacuum(vacuumQueue, requestDeleteStagedBlobFile);
                }
                // schedule disk persist of the new value
                persistQueue.put(Tuple(blobId, blobReference, 1));
                // update memory with the new value
                blobReference.count.set(1);
                // done!
                return Tuple(blobId, staged._2);
            }
            final int count = blobReference.count.get();
            // guard against a concurrent #add or #remove
            if (count == PROMOTING || count == UPDATING_COUNT) {
                // spin whilst another thread promotes the blob, or updates the reference count
                // sleep a small time to save CPU
                Thread.sleep(10);
                continue;
            }
            // i.e. wait for the deletion of the blob to complete, and then we can add the blob again
            if (count == DELETING) {
                blobReference = references.computeIfAbsent(blobId, k -> new BlobReference(STAGED));
                // loop again
                continue;
            }
            // only increment the blob reference if the blob is active!
            if (count >= 0 && blobReference.count.compareAndSet(count, UPDATING_COUNT)) {
                // NOTE: we are the only thread that can be in this branch for the blobId
                final int newCount = count + 1;
                // write journal entries to the WAL
                if (journalManager != null) {
                    try {
                        journalManager.journal(new UpdateBlobRefCountLoggable(transaction.getId(), blobId, count, newCount));
                        // force WAL entries to disk!
                        journalManager.flush(true, true);
                    } catch (final JournalException e) {
                        // restore the state of the blobReference first!
                        blobReference.count.set(count);
                        throw new IOException(e);
                    }
                }
                // persist the new value
                persistQueue.put(Tuple(blobId, blobReference, newCount));
                // update memory with the new value, and release other spinning threads
                blobReference.count.set(newCount);
                // done!
                return Tuple(blobId, staged._2);
            }
        }
    } catch (final InterruptedException e) {
        // thrown by persistQueue.put or Thread.sleep
        Thread.currentThread().interrupt();
        throw new IOException(e);
    }
}
Also used : Path(java.nio.file.Path) Tuple3(com.evolvedbinary.j8fu.tuple.Tuple3) Tuple2(com.evolvedbinary.j8fu.tuple.Tuple2) Txn(org.exist.storage.txn.Txn) JournalException(org.exist.storage.journal.JournalException) RawDataBackup(org.exist.backup.RawDataBackup) ByteBuffer(java.nio.ByteBuffer) FileUtils(org.exist.util.FileUtils) Tuple(com.evolvedbinary.j8fu.tuple.Tuple.Tuple) ThreadUtils.nameInstanceThread(org.exist.util.ThreadUtils.nameInstanceThread) DigestInputStream(org.exist.util.crypto.digest.DigestInputStream) StreamableDigest(org.exist.util.crypto.digest.StreamableDigest) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) DigestType(org.exist.util.crypto.digest.DigestType) LOG_UPDATE_BLOB_REF_COUNT(org.exist.storage.blob.BlobLoggable.LOG_UPDATE_BLOB_REF_COUNT) Path(java.nio.file.Path) java.util.concurrent(java.util.concurrent) StandardOpenOption(java.nio.file.StandardOpenOption) FileNotFoundException(java.io.FileNotFoundException) SeekableByteChannel(java.nio.channels.SeekableByteChannel) Logger(org.apache.logging.log4j.Logger) TxnListener(org.exist.storage.txn.TxnListener) LogEntryTypes(org.exist.storage.journal.LogEntryTypes) java.util(java.util) Try(com.evolvedbinary.j8fu.Try) ThreadSafe(net.jcip.annotations.ThreadSafe) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AtomicReference(java.util.concurrent.atomic.AtomicReference) Function(java.util.function.Function) FilterInputStream(java.io.FilterInputStream) LOG_STORE_BLOB_FILE(org.exist.storage.blob.BlobLoggable.LOG_STORE_BLOB_FILE) ThreadUtils.newInstanceSubThreadGroup(org.exist.util.ThreadUtils.newInstanceSubThreadGroup) LogException(org.exist.storage.journal.LogException) REPLACE_EXISTING(java.nio.file.StandardCopyOption.REPLACE_EXISTING) Nullable(javax.annotation.Nullable) OutputStream(java.io.OutputStream) Database(org.exist.Database) ATOMIC_MOVE(java.nio.file.StandardCopyOption.ATOMIC_MOVE) CountingInputStream(org.apache.commons.io.input.CountingInputStream) Files(java.nio.file.Files) HexEncoder.bytesToHex(org.exist.util.HexEncoder.bytesToHex) JournalManager(org.exist.storage.journal.JournalManager) IOException(java.io.IOException) BlobReference(org.exist.storage.blob.BlobStoreImpl.BlobReference) UUIDGenerator(org.exist.util.UUIDGenerator) MessageDigest(org.exist.util.crypto.digest.MessageDigest) FileUtils.fileName(org.exist.util.FileUtils.fileName) TaggedTryUnchecked(com.evolvedbinary.j8fu.Try.TaggedTryUnchecked) LogManager(org.apache.logging.log4j.LogManager) InputStream(java.io.InputStream) BlobReference(org.exist.storage.blob.BlobStoreImpl.BlobReference) JournalException(org.exist.storage.journal.JournalException) JournalManager(org.exist.storage.journal.JournalManager) IOException(java.io.IOException) MessageDigest(org.exist.util.crypto.digest.MessageDigest)

Example 2 with Tuple3

use of com.evolvedbinary.j8fu.tuple.Tuple3 in project exist by eXist-db.

the class LockManager method acquireDocumentWriteLock.

/**
 * Acquire a WRITE_LOCK on a Document
 *
 * @param documentPath The URI of the Document within the database
 *
 * @return the lock for the Document
 *
 * @throws LockException if the lock could not be acquired
 */
public ManagedDocumentLock acquireDocumentWriteLock(final XmldbURI documentPath) throws LockException {
    if (usePathLocksForDocuments) {
        final LockGroup lockGroup = acquirePathWriteLock(LockType.DOCUMENT, documentPath, false);
        return new ManagedDocumentLock(documentPath, Arrays.stream(lockGroup.locks).map(Tuple3::get_1).toArray(MultiLock[]::new), () -> unlockAll(lockGroup.locks, l -> lockTable.released(lockGroup.groupId, l._3, LockType.DOCUMENT, l._2)));
    } else {
        final long groupId = System.nanoTime();
        final String path = documentPath.toString();
        final MultiLock lock = getDocumentLock(path);
        lockTable.attempt(groupId, path, LockType.DOCUMENT, Lock.LockMode.WRITE_LOCK);
        if (lock(lock, Lock.LockMode.WRITE_LOCK)) {
            lockTable.acquired(groupId, path, LockType.DOCUMENT, Lock.LockMode.WRITE_LOCK);
        } else {
            lockTable.attemptFailed(groupId, path, LockType.DOCUMENT, Lock.LockMode.WRITE_LOCK);
            throw new LockException("Unable to acquire WRITE_LOCK for: " + path);
        }
        return new ManagedDocumentLock(documentPath, lock, () -> {
            lock.asWriteLock().unlock();
            lockTable.released(groupId, path, LockType.DOCUMENT, Lock.LockMode.WRITE_LOCK);
        });
    }
}
Also used : Tuple3(com.evolvedbinary.j8fu.tuple.Tuple3) Consumer(java.util.function.Consumer) Arrays(java.util.Arrays) Logger(org.apache.logging.log4j.Logger) ReentrantLock(java.util.concurrent.locks.ReentrantLock) WeakLazyStripes(org.exist.util.WeakLazyStripes) LockException(org.exist.util.LockException) XmldbURI(org.exist.xmldb.XmldbURI) LockType(org.exist.storage.lock.Lock.LockType) Configuration(org.exist.util.Configuration) MultiLock(uk.ac.ic.doc.slurp.multilock.MultiLock) LogManager(org.apache.logging.log4j.LogManager) LockException(org.exist.util.LockException) Tuple3(com.evolvedbinary.j8fu.tuple.Tuple3) MultiLock(uk.ac.ic.doc.slurp.multilock.MultiLock)

Example 3 with Tuple3

use of com.evolvedbinary.j8fu.tuple.Tuple3 in project exist by eXist-db.

the class LockManager method acquireDocumentReadLock.

/**
 * Acquire a READ_LOCK on a Document
 *
 * @param documentPath The URI of the Document within the database
 *
 * @return the lock for the Document
 *
 * @throws LockException if the lock could not be acquired
 */
public ManagedDocumentLock acquireDocumentReadLock(final XmldbURI documentPath) throws LockException {
    if (usePathLocksForDocuments) {
        final LockGroup lockGroup = acquirePathReadLock(LockType.DOCUMENT, documentPath);
        return new ManagedDocumentLock(documentPath, Arrays.stream(lockGroup.locks).map(Tuple3::get_1).toArray(MultiLock[]::new), () -> unlockAll(lockGroup.locks, l -> lockTable.released(lockGroup.groupId, l._3, LockType.DOCUMENT, l._2)));
    } else {
        final long groupId = System.nanoTime();
        final String path = documentPath.toString();
        final MultiLock lock = getDocumentLock(path);
        lockTable.attempt(groupId, path, LockType.DOCUMENT, Lock.LockMode.READ_LOCK);
        if (lock(lock, Lock.LockMode.READ_LOCK)) {
            lockTable.acquired(groupId, path, LockType.DOCUMENT, Lock.LockMode.READ_LOCK);
        } else {
            lockTable.attemptFailed(groupId, path, LockType.DOCUMENT, Lock.LockMode.READ_LOCK);
            throw new LockException("Unable to acquire READ_LOCK for: " + path);
        }
        return new ManagedDocumentLock(documentPath, lock, () -> {
            lock.asReadLock().unlock();
            lockTable.released(groupId, path, LockType.DOCUMENT, Lock.LockMode.READ_LOCK);
        });
    }
}
Also used : Tuple3(com.evolvedbinary.j8fu.tuple.Tuple3) Consumer(java.util.function.Consumer) Arrays(java.util.Arrays) Logger(org.apache.logging.log4j.Logger) ReentrantLock(java.util.concurrent.locks.ReentrantLock) WeakLazyStripes(org.exist.util.WeakLazyStripes) LockException(org.exist.util.LockException) XmldbURI(org.exist.xmldb.XmldbURI) LockType(org.exist.storage.lock.Lock.LockType) Configuration(org.exist.util.Configuration) MultiLock(uk.ac.ic.doc.slurp.multilock.MultiLock) LogManager(org.apache.logging.log4j.LogManager) LockException(org.exist.util.LockException) Tuple3(com.evolvedbinary.j8fu.tuple.Tuple3) MultiLock(uk.ac.ic.doc.slurp.multilock.MultiLock)

Example 4 with Tuple3

use of com.evolvedbinary.j8fu.tuple.Tuple3 in project exist by eXist-db.

the class LockManager method acquirePathWriteLock.

/**
 * Acquires a WRITE_LOCK on a path (and implicitly all descendant paths).
 *
 * @param path The path for which a lock is requested.
 * @param lockParent true if we should also explicitly write lock the parent path.
 *
 * @return A WRITE_LOCK on the path.
 */
LockGroup acquirePathWriteLock(final LockType lockType, final XmldbURI path, final boolean lockParent) throws LockException {
    final XmldbURI[] segments = path.getPathSegments();
    final long groupId = System.nanoTime();
    String pathStr = "";
    final Tuple3<MultiLock, Lock.LockMode, String>[] locked = new Tuple3[segments.length];
    for (int i = 0; i < segments.length; i++) {
        pathStr += '/' + segments[i].toString();
        final Lock.LockMode lockMode;
        if (lockParent && i + 2 == segments.length) {
            // parent
            lockMode = Lock.LockMode.WRITE_LOCK;
        } else if (i + 1 == segments.length) {
            // leaf
            lockMode = Lock.LockMode.WRITE_LOCK;
        } else {
            if (!pathsMultiWriter) {
                // single-writer/multi-reader
                lockMode = Lock.LockMode.WRITE_LOCK;
            } else {
                // multi-writer/multi-reader
                lockMode = Lock.LockMode.INTENTION_WRITE;
            }
        }
        final MultiLock lock = getPathLock(pathStr);
        if (upgradeCheck && lockMode == Lock.LockMode.WRITE_LOCK && (lock.getIntentionReadHoldCount() > 0 || lock.getReadHoldCount() > 0)) {
            throw new LockException("Lock upgrading would lead to a self-deadlock: " + pathStr);
        }
        if (warnWaitOnReadForWrite && lockMode == Lock.LockMode.WRITE_LOCK) {
            if (lock.getIntentionReadLockCount() > 0) {
                LOG.warn("About to acquire WRITE_LOCK for: {}, but INTENTION_READ_LOCK held by other thread(s): ", pathStr);
            } else if (lock.getReadLockCount() > 0) {
                LOG.warn("About to acquire WRITE_LOCK for: {}, but READ_LOCK held by other thread(s): ", pathStr);
            }
        }
        lockTable.attempt(groupId, pathStr, lockType, lockMode);
        if (lock(lock, lockMode)) {
            locked[i] = new Tuple3<>(lock, lockMode, pathStr);
            lockTable.acquired(groupId, pathStr, lockType, lockMode);
        } else {
            lockTable.attemptFailed(groupId, pathStr, lockType, lockMode);
            unlockAll(locked, l -> lockTable.released(groupId, l._3, lockType, l._2));
            throw new LockException("Unable to acquire " + lockType + " " + lockMode + " for: " + pathStr);
        }
    }
    return new LockGroup(groupId, locked);
}
Also used : ReentrantLock(java.util.concurrent.locks.ReentrantLock) MultiLock(uk.ac.ic.doc.slurp.multilock.MultiLock) LockException(org.exist.util.LockException) Tuple3(com.evolvedbinary.j8fu.tuple.Tuple3) MultiLock(uk.ac.ic.doc.slurp.multilock.MultiLock) XmldbURI(org.exist.xmldb.XmldbURI)

Example 5 with Tuple3

use of com.evolvedbinary.j8fu.tuple.Tuple3 in project exist by eXist-db.

the class LockManager method acquirePathReadLock.

/**
 * Acquires a READ_LOCK on a database path (and implicitly all descendant paths).
 *
 * @param lockType The type of the lock
 * @param path The path for which a lock is requested.
 *
 * @return A READ_LOCK on the Collection.
 *
 * @throws LockException if a lock error occurs
 */
public LockGroup acquirePathReadLock(final LockType lockType, final XmldbURI path) throws LockException {
    final XmldbURI[] segments = path.getPathSegments();
    final long groupId = System.nanoTime();
    String pathStr = "";
    final Tuple3<MultiLock, Lock.LockMode, String>[] locked = new Tuple3[segments.length];
    for (int i = 0; i < segments.length; i++) {
        pathStr += '/' + segments[i].toString();
        final Lock.LockMode lockMode;
        if (i + 1 == segments.length) {
            // leaf
            lockMode = Lock.LockMode.READ_LOCK;
        } else {
            // ancestor
            lockMode = Lock.LockMode.INTENTION_READ;
        }
        final MultiLock lock = getPathLock(pathStr);
        lockTable.attempt(groupId, pathStr, lockType, lockMode);
        if (lock(lock, lockMode)) {
            locked[i] = new Tuple3<>(lock, lockMode, pathStr);
            lockTable.acquired(groupId, pathStr, lockType, lockMode);
        } else {
            lockTable.attemptFailed(groupId, pathStr, lockType, lockMode);
            unlockAll(locked, l -> lockTable.released(groupId, l._3, lockType, l._2));
            throw new LockException("Unable to acquire " + lockType + " " + lockMode + " for: " + pathStr);
        }
    }
    return new LockGroup(groupId, locked);
}
Also used : ReentrantLock(java.util.concurrent.locks.ReentrantLock) MultiLock(uk.ac.ic.doc.slurp.multilock.MultiLock) LockException(org.exist.util.LockException) Tuple3(com.evolvedbinary.j8fu.tuple.Tuple3) MultiLock(uk.ac.ic.doc.slurp.multilock.MultiLock) XmldbURI(org.exist.xmldb.XmldbURI)

Aggregations

Tuple3 (com.evolvedbinary.j8fu.tuple.Tuple3)7 LockException (org.exist.util.LockException)5 XmldbURI (org.exist.xmldb.XmldbURI)5 ReentrantLock (java.util.concurrent.locks.ReentrantLock)4 MultiLock (uk.ac.ic.doc.slurp.multilock.MultiLock)4 LogManager (org.apache.logging.log4j.LogManager)3 Logger (org.apache.logging.log4j.Logger)3 Tuple2 (com.evolvedbinary.j8fu.tuple.Tuple2)2 Arrays (java.util.Arrays)2 Consumer (java.util.function.Consumer)2 Function (java.util.function.Function)2 Collection (org.exist.collections.Collection)2 LockType (org.exist.storage.lock.Lock.LockType)2 Configuration (org.exist.util.Configuration)2 WeakLazyStripes (org.exist.util.WeakLazyStripes)2 Try (com.evolvedbinary.j8fu.Try)1 TaggedTryUnchecked (com.evolvedbinary.j8fu.Try.TaggedTryUnchecked)1 Tuple (com.evolvedbinary.j8fu.tuple.Tuple.Tuple)1 FileNotFoundException (java.io.FileNotFoundException)1 FilterInputStream (java.io.FilterInputStream)1