Examples with Hash org.projectnessie.versioned.Hash used on opensource projects

Example 26 with Hash

use of org.projectnessie.versioned.Hash in project nessie by projectnessie.

the class AbstractDatabaseAdapter method transplantAttempt.

/**
 * Logic implementation of a transplant-attempt.
 *
 * @param ctx technical operation context
 * @param targetBranch target reference with expected HEAD
 * @param expectedHead if present, {@code targetBranch}'s current HEAD must be equal to this value
 * @param targetHead current HEAD of {@code targetBranch}
 * @param sequenceToTransplant sequential list of commits to transplant from {@code source}
 * @param branchCommits consumer for the individual commits to merge
 * @param newKeyLists consumer for optimistically written {@link KeyListEntity}s
 * @param rewriteMetadata function to rewrite the commit-metadata for copied commits
 * @return hash of the last commit-log-entry written to {@code targetBranch}
 */
protected Hash transplantAttempt(OP_CONTEXT ctx, long timeInMicros, BranchName targetBranch, Optional<Hash> expectedHead, Hash targetHead, List<Hash> sequenceToTransplant, Consumer<Hash> branchCommits, Consumer<Hash> newKeyLists, Function<ByteString, ByteString> rewriteMetadata) throws ReferenceNotFoundException, ReferenceConflictException {
    if (sequenceToTransplant.isEmpty()) {
        throw new IllegalArgumentException("No hashes to transplant given.");
    }
    // 1. ensure 'expectedHash' is a parent of HEAD-of-'targetBranch' & collect keys
    List<CommitLogEntry> targetEntriesReverseChronological = new ArrayList<>();
    hashOnRef(ctx, targetHead, targetBranch, expectedHead, targetEntriesReverseChronological::add);
    // Exclude the expected-hash on the target-branch from key-collisions check
    if (!targetEntriesReverseChronological.isEmpty() && expectedHead.isPresent() && targetEntriesReverseChronological.get(0).getHash().equals(expectedHead.get())) {
        targetEntriesReverseChronological.remove(0);
    }
    Collections.reverse(targetEntriesReverseChronological);
    // 2. Collect modified keys.
    Set<Key> keysTouchedOnTarget = collectModifiedKeys(targetEntriesReverseChronological);
    // 4. ensure 'sequenceToTransplant' is sequential
    int[] index = new int[] { sequenceToTransplant.size() - 1 };
    Hash lastHash = sequenceToTransplant.get(sequenceToTransplant.size() - 1);
    List<CommitLogEntry> commitsToTransplantChronological = takeUntilExcludeLast(readCommitLogStream(ctx, lastHash), e -> {
        int i = index[0]--;
        if (i == -1) {
            return true;
        }
        if (!e.getHash().equals(sequenceToTransplant.get(i))) {
            throw new IllegalArgumentException("Sequence of hashes is not contiguous.");
        }
        return false;
    }).collect(Collectors.toList());
    // 5. check for key-collisions
    checkForKeyCollisions(ctx, targetHead, keysTouchedOnTarget, commitsToTransplantChronological);
    // (no need to verify the global states during a transplant)
    // 6. re-apply commits in 'sequenceToTransplant' onto 'targetBranch'
    targetHead = copyCommits(ctx, timeInMicros, targetHead, commitsToTransplantChronological, newKeyLists, rewriteMetadata);
    // 7. Write commits
    commitsToTransplantChronological.stream().map(CommitLogEntry::getHash).forEach(branchCommits);
    writeMultipleCommits(ctx, commitsToTransplantChronological);
    return targetHead;
}

Example 27 with Hash

use of org.projectnessie.versioned.Hash in project nessie by projectnessie.

the class AbstractDatabaseAdapter method checkForKeyCollisions.

/**
 * For merge/transplant, verifies that the given commits do not touch any of the given keys.
 *
 * @param commitsChronological list of commit-log-entries, in order of commit-operations,
 *     chronological order
 */
protected void checkForKeyCollisions(OP_CONTEXT ctx, Hash refHead, Set<Key> keysTouchedOnTarget, List<CommitLogEntry> commitsChronological) throws ReferenceConflictException, ReferenceNotFoundException {
    Set<Key> keyCollisions = new HashSet<>();
    for (int i = commitsChronological.size() - 1; i >= 0; i--) {
        CommitLogEntry sourceCommit = commitsChronological.get(i);
        Stream.concat(sourceCommit.getPuts().stream().map(KeyWithBytes::getKey), sourceCommit.getDeletes().stream()).filter(keysTouchedOnTarget::contains).forEach(keyCollisions::add);
    }
    if (!keyCollisions.isEmpty()) {
        removeKeyCollisionsForNamespaces(ctx, refHead, commitsChronological.get(commitsChronological.size() - 1).getHash(), keyCollisions);
        if (!keyCollisions.isEmpty()) {
            throw new ReferenceConflictException(String.format("The following keys have been changed in conflict: %s", keyCollisions.stream().map(k -> String.format("'%s'", k.toString())).collect(Collectors.joining(", "))));
        }
    }
}

Example 28 with Hash

use of org.projectnessie.versioned.Hash in project nessie by projectnessie.

the class NonTransactionalDatabaseAdapter method writeRefLogEntry.

protected RefLogEntry writeRefLogEntry(NonTransactionalOperationContext ctx, GlobalStatePointer pointer, String refName, RefType refType, Hash commitHash, Operation operation, long timeInMicros, List<Hash> sourceHashes) throws ReferenceConflictException {
    Hash parentHash = Hash.of(pointer.getRefLogId());
    Hash currentRefLogId = randomHash();
    Stream<ByteString> newParents;
    if (pointer.getRefLogParentsInclHeadCount() == 0 || !pointer.getRefLogId().equals(pointer.getRefLogParentsInclHead(0))) {
        // Before Nessie 0.21.0
        newParents = Stream.of(parentHash.asBytes());
        RefLog currentEntry = fetchFromRefLog(ctx, parentHash);
        if (currentEntry != null) {
            newParents = Stream.concat(newParents, currentEntry.getParents().stream().limit(config.getParentsPerRefLogEntry() - 1).map(Hash::asBytes));
        }
    } else {
        // Since Nessie 0.21.0
        newParents = pointer.getRefLogParentsInclHeadList().stream().limit(config.getParentsPerRefLogEntry());
    }
    RefLogEntry.Builder entry = RefLogEntry.newBuilder().setRefLogId(currentRefLogId.asBytes()).setRefName(ByteString.copyFromUtf8(refName)).setRefType(refType).setCommitHash(commitHash.asBytes()).setOperationTime(timeInMicros).setOperation(operation);
    sourceHashes.forEach(hash -> entry.addSourceHashes(hash.asBytes()));
    newParents.forEach(entry::addParents);
    RefLogEntry refLogEntry = entry.build();
    writeRefLog(ctx, refLogEntry);
    return refLogEntry;
}

Example 29 with Hash

use of org.projectnessie.versioned.Hash in project nessie by projectnessie.

the class NonTransactionalDatabaseAdapter method transplant.

@SuppressWarnings("RedundantThrows")
@Override
public Hash transplant(BranchName targetBranch, Optional<Hash> expectedHead, List<Hash> sequenceToTransplant, Function<ByteString, ByteString> updateCommitMetadata) throws ReferenceNotFoundException, ReferenceConflictException {
    try {
        return casOpLoop("transplant", targetBranch, CasOpVariant.COMMIT, (ctx, pointer, branchCommits, newKeyLists) -> {
            Hash targetHead = branchHead(pointer, targetBranch);
            long timeInMicros = commitTimeInMicros();
            targetHead = transplantAttempt(ctx, timeInMicros, targetBranch, expectedHead, targetHead, sequenceToTransplant, branchCommits, newKeyLists, updateCommitMetadata);
            GlobalStateLogEntry newGlobalHead = writeGlobalCommit(ctx, timeInMicros, pointer, Collections.emptyList());
            RefLogEntry newRefLog = writeRefLogEntry(ctx, pointer, targetBranch.getName(), RefLogEntry.RefType.Branch, targetHead, RefLogEntry.Operation.TRANSPLANT, timeInMicros, sequenceToTransplant);
            // Return hash of last commit (targetHead) added to 'targetBranch' (via the casOpLoop)
            return updateGlobalStatePointer(targetBranch, pointer, targetHead, newGlobalHead, newRefLog);
        }, () -> transplantConflictMessage("Retry-failure", targetBranch, expectedHead, sequenceToTransplant));
    } catch (ReferenceNotFoundException | ReferenceConflictException | RuntimeException e) {
        throw e;
    } catch (Exception e) {
        throw new RuntimeException(e);
    }
}

Example 30 with Hash

use of org.projectnessie.versioned.Hash in project nessie by projectnessie.

the class NonTransactionalDatabaseAdapter method compactGlobalLog.

protected Map<String, String> compactGlobalLog(GlobalLogCompactionParams globalLogCompactionParams) {
    if (!globalLogCompactionParams.isEnabled()) {
        return ImmutableMap.of("compacted", "false", "reason", "not enabled");
    }
    // Not using casOpLoop() here, as it is simpler than adopting casOpLoop().
    try (TryLoopState tryState = newTryLoopState("compact-global-log", ts -> repoDescUpdateConflictMessage(String.format("%s after %d retries, %d ms", "Retry-Failure", ts.getRetries(), ts.getDuration(TimeUnit.MILLISECONDS))), this::tryLoopStateCompletion, config)) {
        CompactionStats stats = new CompactionStats();
        while (true) {
            NonTransactionalOperationContext ctx = NON_TRANSACTIONAL_OPERATION_CONTEXT;
            GlobalStatePointer pointer = fetchGlobalPointer(ctx);
            // Collect the old global-log-ids, to delete those after compaction
            List<Hash> oldLogIds = new ArrayList<>();
            // Map with all global contents.
            Map<String, ByteString> globalContents = new HashMap<>();
            // Content-IDs, most recently updated contents first.
            List<String> contentIdsByRecency = new ArrayList<>();
            // Read the global log - from the most recent global-log entry to the oldest.
            try (Stream<GlobalStateLogEntry> globalLog = globalLogFetcher(ctx, pointer)) {
                globalLog.forEach(e -> {
                    if (stats.read < globalLogCompactionParams.getNoCompactionWhenCompactedWithin() && stats.puts > stats.read) {
                        // do not compact.
                        throw COMPACTION_NOT_NECESSARY_WITHIN;
                    }
                    stats.read++;
                    oldLogIds.add(Hash.of(e.getId()));
                    for (ContentIdWithBytes put : e.getPutsList()) {
                        stats.puts++;
                        String cid = put.getContentId().getId();
                        if (globalContents.putIfAbsent(cid, put.getValue()) == null) {
                            stats.uniquePuts++;
                            contentIdsByRecency.add(cid);
                        }
                    }
                });
                if (stats.read < globalLogCompactionParams.getNoCompactionUpToLength()) {
                    // single-bulk read, so do not compact at all.
                    throw COMPACTION_NOT_NECESSARY_LENGTH;
                }
            } catch (RuntimeException e) {
                if (e == COMPACTION_NOT_NECESSARY_WITHIN) {
                    tryState.success(null);
                    return ImmutableMap.of("compacted", "false", "reason", String.format("compacted entry within %d most recent log entries", globalLogCompactionParams.getNoCompactionWhenCompactedWithin()));
                }
                if (e == COMPACTION_NOT_NECESSARY_LENGTH) {
                    tryState.success(null);
                    return ImmutableMap.of("compacted", "false", "reason", String.format("less than %d entries", globalLogCompactionParams.getNoCompactionUpToLength()));
                }
                throw e;
            }
            // Collect the IDs of the written global-log-entries, to delete those when the CAS
            // operation failed
            List<ByteString> newLogIds = new ArrayList<>();
            // Reverse the order of content-IDs, most recently updated contents LAST.
            // Do this to have the active contents closer to the HEAD of the global log.
            Collections.reverse(contentIdsByRecency);
            // Maintain the list of global-log-entry parent IDs, but in reverse order as in
            // GlobalLogEntry for easier management here.
            List<ByteString> globalParentsReverse = new ArrayList<>(config.getParentsPerGlobalCommit());
            globalParentsReverse.add(NO_ANCESTOR.asBytes());
            GlobalStateLogEntry.Builder currentEntry = newGlobalLogEntryBuilder(commitTimeInMicros()).addParents(globalParentsReverse.get(0));
            for (String cid : contentIdsByRecency) {
                if (currentEntry.buildPartial().getSerializedSize() >= config.getGlobalLogEntrySize()) {
                    compactGlobalLogWriteEntry(ctx, stats, globalParentsReverse, currentEntry, newLogIds);
                    // Prepare new entry
                    currentEntry = newGlobalLogEntryBuilder(commitTimeInMicros());
                    for (int i = globalParentsReverse.size() - 1; i >= 0; i--) {
                        currentEntry.addParents(globalParentsReverse.get(i));
                    }
                }
                ByteString value = globalContents.get(cid);
                currentEntry.addPuts(ContentIdWithBytes.newBuilder().setContentId(AdapterTypes.ContentId.newBuilder().setId(cid)).setTypeUnused(0).setValue(value).build());
            }
            compactGlobalLogWriteEntry(ctx, stats, globalParentsReverse, currentEntry, newLogIds);
            GlobalStatePointer newPointer = GlobalStatePointer.newBuilder().addAllNamedReferences(pointer.getNamedReferencesList()).addAllRefLogParentsInclHead(pointer.getRefLogParentsInclHeadList()).setRefLogId(pointer.getRefLogId()).setGlobalId(currentEntry.getId()).addGlobalParentsInclHead(currentEntry.getId()).addAllGlobalParentsInclHead(currentEntry.getParentsList()).build();
            stats.addToTotal();
            // CAS global pointer
            if (globalPointerCas(ctx, pointer, newPointer)) {
                tryState.success(null);
                cleanUpGlobalLog(ctx, oldLogIds);
                return stats.asMap(tryState);
            }
            // Note: if it turns out that there are too many CAS retries happening, the overall
            // mechanism can be updated as follows. Since the approach below is much more complex
            // and harder to test, if's not part of the initial implementation.
            // 
            // 1. Read the whole global-log as currently, but outside the actual CAS-loop.
            // Save the current HEAD of the global-log
            // 2. CAS-loop:
            // 2.1. Construct and write the new global-log
            // 2.2. Try the CAS, if it succeeds, fine
            // 2.3. If the CAS failed:
            // 2.3.1. Clean up the optimistically written new global-log
            // 2.3.2. Read the global-log from its new HEAD up to the current HEAD from step 1.
            // Only add the most-recent values for the content-IDs in the incrementally
            // read global-log
            // 2.3.3. Remember the "new HEAD" as the "current HEAD"
            // 2.3.4. Continue to step 2.1.
            cleanUpGlobalLog(ctx, newLogIds.stream().map(Hash::of).collect(Collectors.toList()));
            stats.onRetry();
            tryState.retry();
        }
    } catch (ReferenceConflictException e) {
        throw new RuntimeException(e);
    }
}