Examples with GlobalStatePointer org.projectnessie.versioned.persist.serialize.AdapterTypes.GlobalStatePointer used on opensource projects

Example 1 with GlobalStatePointer

use of org.projectnessie.versioned.persist.serialize.AdapterTypes.GlobalStatePointer in project nessie by projectnessie.

the class NonTransactionalDatabaseAdapter method compactGlobalLog.

protected Map<String, String> compactGlobalLog(GlobalLogCompactionParams globalLogCompactionParams) {
    if (!globalLogCompactionParams.isEnabled()) {
        return ImmutableMap.of("compacted", "false", "reason", "not enabled");
    }
    // Not using casOpLoop() here, as it is simpler than adopting casOpLoop().
    try (TryLoopState tryState = newTryLoopState("compact-global-log", ts -> repoDescUpdateConflictMessage(String.format("%s after %d retries, %d ms", "Retry-Failure", ts.getRetries(), ts.getDuration(TimeUnit.MILLISECONDS))), this::tryLoopStateCompletion, config)) {
        CompactionStats stats = new CompactionStats();
        while (true) {
            NonTransactionalOperationContext ctx = NON_TRANSACTIONAL_OPERATION_CONTEXT;
            GlobalStatePointer pointer = fetchGlobalPointer(ctx);
            // Collect the old global-log-ids, to delete those after compaction
            List<Hash> oldLogIds = new ArrayList<>();
            // Map with all global contents.
            Map<String, ByteString> globalContents = new HashMap<>();
            // Content-IDs, most recently updated contents first.
            List<String> contentIdsByRecency = new ArrayList<>();
            // Read the global log - from the most recent global-log entry to the oldest.
            try (Stream<GlobalStateLogEntry> globalLog = globalLogFetcher(ctx, pointer)) {
                globalLog.forEach(e -> {
                    if (stats.read < globalLogCompactionParams.getNoCompactionWhenCompactedWithin() && stats.puts > stats.read) {
                        // do not compact.
                        throw COMPACTION_NOT_NECESSARY_WITHIN;
                    }
                    stats.read++;
                    oldLogIds.add(Hash.of(e.getId()));
                    for (ContentIdWithBytes put : e.getPutsList()) {
                        stats.puts++;
                        String cid = put.getContentId().getId();
                        if (globalContents.putIfAbsent(cid, put.getValue()) == null) {
                            stats.uniquePuts++;
                            contentIdsByRecency.add(cid);
                        }
                    }
                });
                if (stats.read < globalLogCompactionParams.getNoCompactionUpToLength()) {
                    // single-bulk read, so do not compact at all.
                    throw COMPACTION_NOT_NECESSARY_LENGTH;
                }
            } catch (RuntimeException e) {
                if (e == COMPACTION_NOT_NECESSARY_WITHIN) {
                    tryState.success(null);
                    return ImmutableMap.of("compacted", "false", "reason", String.format("compacted entry within %d most recent log entries", globalLogCompactionParams.getNoCompactionWhenCompactedWithin()));
                }
                if (e == COMPACTION_NOT_NECESSARY_LENGTH) {
                    tryState.success(null);
                    return ImmutableMap.of("compacted", "false", "reason", String.format("less than %d entries", globalLogCompactionParams.getNoCompactionUpToLength()));
                }
                throw e;
            }
            // Collect the IDs of the written global-log-entries, to delete those when the CAS
            // operation failed
            List<ByteString> newLogIds = new ArrayList<>();
            // Reverse the order of content-IDs, most recently updated contents LAST.
            // Do this to have the active contents closer to the HEAD of the global log.
            Collections.reverse(contentIdsByRecency);
            // Maintain the list of global-log-entry parent IDs, but in reverse order as in
            // GlobalLogEntry for easier management here.
            List<ByteString> globalParentsReverse = new ArrayList<>(config.getParentsPerGlobalCommit());
            globalParentsReverse.add(NO_ANCESTOR.asBytes());
            GlobalStateLogEntry.Builder currentEntry = newGlobalLogEntryBuilder(commitTimeInMicros()).addParents(globalParentsReverse.get(0));
            for (String cid : contentIdsByRecency) {
                if (currentEntry.buildPartial().getSerializedSize() >= config.getGlobalLogEntrySize()) {
                    compactGlobalLogWriteEntry(ctx, stats, globalParentsReverse, currentEntry, newLogIds);
                    // Prepare new entry
                    currentEntry = newGlobalLogEntryBuilder(commitTimeInMicros());
                    for (int i = globalParentsReverse.size() - 1; i >= 0; i--) {
                        currentEntry.addParents(globalParentsReverse.get(i));
                    }
                }
                ByteString value = globalContents.get(cid);
                currentEntry.addPuts(ContentIdWithBytes.newBuilder().setContentId(AdapterTypes.ContentId.newBuilder().setId(cid)).setTypeUnused(0).setValue(value).build());
            }
            compactGlobalLogWriteEntry(ctx, stats, globalParentsReverse, currentEntry, newLogIds);
            GlobalStatePointer newPointer = GlobalStatePointer.newBuilder().addAllNamedReferences(pointer.getNamedReferencesList()).addAllRefLogParentsInclHead(pointer.getRefLogParentsInclHeadList()).setRefLogId(pointer.getRefLogId()).setGlobalId(currentEntry.getId()).addGlobalParentsInclHead(currentEntry.getId()).addAllGlobalParentsInclHead(currentEntry.getParentsList()).build();
            stats.addToTotal();
            // CAS global pointer
            if (globalPointerCas(ctx, pointer, newPointer)) {
                tryState.success(null);
                cleanUpGlobalLog(ctx, oldLogIds);
                return stats.asMap(tryState);
            }
            // Note: if it turns out that there are too many CAS retries happening, the overall
            // mechanism can be updated as follows. Since the approach below is much more complex
            // and harder to test, if's not part of the initial implementation.
            // 
            // 1. Read the whole global-log as currently, but outside the actual CAS-loop.
            // Save the current HEAD of the global-log
            // 2. CAS-loop:
            // 2.1. Construct and write the new global-log
            // 2.2. Try the CAS, if it succeeds, fine
            // 2.3. If the CAS failed:
            // 2.3.1. Clean up the optimistically written new global-log
            // 2.3.2. Read the global-log from its new HEAD up to the current HEAD from step 1.
            // Only add the most-recent values for the content-IDs in the incrementally
            // read global-log
            // 2.3.3. Remember the "new HEAD" as the "current HEAD"
            // 2.3.4. Continue to step 2.1.
            cleanUpGlobalLog(ctx, newLogIds.stream().map(Hash::of).collect(Collectors.toList()));
            stats.onRetry();
            tryState.retry();
        }
    } catch (ReferenceConflictException e) {
        throw new RuntimeException(e);
    }
}

Example 2 with GlobalStatePointer

use of org.projectnessie.versioned.persist.serialize.AdapterTypes.GlobalStatePointer in project nessie by projectnessie.

the class RocksDatabaseAdapter method doGlobalPointerCas.

@Override
protected boolean doGlobalPointerCas(NonTransactionalOperationContext ctx, GlobalStatePointer expected, GlobalStatePointer newPointer) {
    Lock lock = dbInstance.getLock().writeLock();
    lock.lock();
    try {
        byte[] bytes = db.get(dbInstance.getCfGlobalPointer(), globalPointerKey());
        GlobalStatePointer oldPointer = bytes != null ? GlobalStatePointer.parseFrom(bytes) : null;
        if (oldPointer == null || !oldPointer.getGlobalId().equals(expected.getGlobalId())) {
            return false;
        }
        db.put(dbInstance.getCfGlobalPointer(), globalPointerKey(), newPointer.toByteArray());
        return true;
    } catch (InvalidProtocolBufferException | RocksDBException e) {
        throw new RuntimeException(e);
    } finally {
        lock.unlock();
    }
}

Example 3 with GlobalStatePointer

use of org.projectnessie.versioned.persist.serialize.AdapterTypes.GlobalStatePointer in project nessie by projectnessie.

the class NonTransactionalDatabaseAdapter method namedRefs.

@Override
public Stream<ReferenceInfo<ByteString>> namedRefs(GetNamedRefsParams params) throws ReferenceNotFoundException {
    Preconditions.checkNotNull(params, "Parameter for GetNamedRefsParams must not be null.");
    Preconditions.checkArgument(namedRefsAnyRetrieves(params), "Must retrieve branches or tags or both.");
    GlobalStatePointer pointer = fetchGlobalPointer(NON_TRANSACTIONAL_OPERATION_CONTEXT);
    if (pointer == null) {
        return Stream.empty();
    }
    Hash defaultBranchHead = namedRefsDefaultBranchHead(params, pointer);
    Stream<ReferenceInfo<ByteString>> refs = pointer.getNamedReferencesList().stream().map(r -> ReferenceInfo.of(Hash.of(r.getRef().getHash()), toNamedRef(r.getRef().getType(), r.getName())));
    return namedRefsFilterAndEnhance(NON_TRANSACTIONAL_OPERATION_CONTEXT, params, defaultBranchHead, refs);
}

Example 4 with GlobalStatePointer

use of org.projectnessie.versioned.persist.serialize.AdapterTypes.GlobalStatePointer in project nessie by projectnessie.

the class NonTransactionalDatabaseAdapter method casOpLoop.

/**
 * This is the actual CAS-loop, which applies an operation onto a named-ref.
 *
 * @param ref named-reference on which the operation happens
 * @param opVariant influences the behavior, whether the operation adds one or more commits and
 *     whether the operation deletes the named reference.
 * @param casOp the implementation of the CAS-operation
 * @param retryErrorMessage provides an error-message for a {@link ReferenceConflictException}
 *     when the CAS operation failed to complete within the configured time and number of retries.
 */
protected Hash casOpLoop(String opName, NamedRef ref, CasOpVariant opVariant, CasOp casOp, Supplier<String> retryErrorMessage) throws VersionStoreException {
    NonTransactionalOperationContext ctx = NON_TRANSACTIONAL_OPERATION_CONTEXT;
    try (TryLoopState tryState = newTryLoopState(opName, ts -> repoDescUpdateConflictMessage(String.format("%s after %d retries, %d ms", retryErrorMessage.get(), ts.getRetries(), ts.getDuration(TimeUnit.MILLISECONDS))), this::tryLoopStateCompletion, config)) {
        while (true) {
            GlobalStatePointer pointer = fetchGlobalPointer(ctx);
            if (pointer == null) {
                throw referenceNotFound(ref);
            }
            Set<Hash> individualCommits = new HashSet<>();
            Set<Hash> individualKeyLists = new HashSet<>();
            GlobalStatePointer newPointer = casOp.apply(ctx, pointer, individualCommits::add, individualKeyLists::add);
            if (newPointer.getGlobalId().equals(pointer.getGlobalId())) {
                return tryState.success(branchHead(pointer, ref));
            }
            Hash branchHead = opVariant.deleteRef ? null : branchHead(newPointer, ref);
            if (pointer.getGlobalId().equals(newPointer.getGlobalId())) {
                throw hashCollisionDetected();
            }
            if (globalPointerCas(ctx, pointer, newPointer)) {
                return tryState.success(branchHead);
            } else if (opVariant.commitOp) {
                if (branchHead != null) {
                    individualCommits.add(branchHead);
                }
                cleanUpCommitCas(ctx, Hash.of(newPointer.getGlobalId()), individualCommits, individualKeyLists, Hash.of(newPointer.getRefLogId()));
            }
            tryState.retry();
        }
    }
}

Example 5 with GlobalStatePointer

use of org.projectnessie.versioned.persist.serialize.AdapterTypes.GlobalStatePointer in project nessie by projectnessie.

the class NonTransactionalDatabaseAdapter method globalLogFetcher.

private Stream<GlobalStateLogEntry> globalLogFetcher(NonTransactionalOperationContext ctx, GlobalStatePointer pointer) {
    if (pointer == null) {
        return Stream.empty();
    }
    // Before Nessie 0.21.0: the global-state-pointer contains the "heads" for the ref-log and
    // global-log, so it has to read the head ref-log & global-log to get the IDs of all the
    // previous parents to fill the parents in the new global-log-entry & ref-log-entry.
    // 
    // Since Nessie 0.21.0: the global-state-pointer contains the "heads" for the ref-log and
    // global-log PLUS the parents of those, so Nessie no longer need to read the head entries
    // from the ref-log + global-log.
    // 
    // The check of the first entry is there to ensure backwards compatibility and also
    // rolling-upgrades work.
    Spliterator<GlobalStateLogEntry> split;
    if (pointer.getGlobalParentsInclHeadCount() == 0 || !pointer.getGlobalId().equals(pointer.getGlobalParentsInclHead(0))) {
        // Before Nessie 0.21.0
        Hash initialId = Hash.of(pointer.getGlobalId());
        GlobalStateLogEntry initial = fetchFromGlobalLog(ctx, initialId);
        if (initial == null) {
            throw new RuntimeException(new ReferenceNotFoundException(String.format("Global log entry '%s' not does not exist.", initialId.asString())));
        }
        split = logFetcher(ctx, initial, this::fetchPageFromGlobalLog, e -> e.getParentsList().stream().map(Hash::of).collect(Collectors.toList()));
    } else {
        // Since Nessie 0.21.0
        List<Hash> hashes = pointer.getGlobalParentsInclHeadList().stream().map(Hash::of).collect(Collectors.toList());
        split = logFetcherWithPage(ctx, hashes, this::fetchPageFromGlobalLog, e -> e.getParentsList().stream().map(Hash::of).collect(Collectors.toList()));
    }
    return StreamSupport.stream(split, false);
}