Examples with Hash org.projectnessie.versioned.Hash used on opensource projects

Example 96 with Hash

use of org.projectnessie.versioned.Hash in project nessie by projectnessie.

the class AbstractGlobalStates method globalStates.

/**
 * Rudimentary test for Nessie-GC related basic operations to collect all globally known keys and
 * the global-state-logs.
 */
@ParameterizedTest
@MethodSource("globalStatesParams")
void globalStates(GlobalStateParam param) throws Exception {
    List<BranchName> branches = IntStream.range(0, param.branches).mapToObj(i -> BranchName.of("globalStates-" + i)).collect(Collectors.toList());
    Map<BranchName, Hash> heads = branches.stream().collect(Collectors.toMap(b -> b, b -> catchingFunction(() -> databaseAdapter.create(b, databaseAdapter.hashOnReference(BranchName.of("main"), Optional.empty())))));
    Map<ContentId, ByteString> currentStates = new HashMap<>();
    Set<Key> keys = IntStream.range(0, param.tables).mapToObj(i -> Key.of("table", Integer.toString(i))).collect(Collectors.toSet());
    Set<ContentId> usedContentIds = new HashSet<>();
    Map<ContentId, ByteString> expectedGlobalStates = new HashMap<>();
    for (int commit = 0; commit < param.commitsPerBranch; commit++) {
        for (BranchName branch : branches) {
            ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder().commitToBranch(branch).expectedHead(Optional.of(heads.get(branch))).commitMetaSerialized(ByteString.copyFromUtf8("some commit#" + commit + " branch " + branch.getName()));
            for (Key key : keys) {
                if (param.tableCommitProbability == 1.0f || ThreadLocalRandom.current().nextDouble(0d, 1d) <= param.tableCommitProbability) {
                    String state = "state-commit-" + commit + "+" + key;
                    String value = "value-commit-" + commit + "+" + key;
                    ContentId contentId = ContentId.of(key.toString() + "-" + branch.getName());
                    ByteString put = ByteString.copyFromUtf8(value);
                    ByteString global = ByteString.copyFromUtf8(state);
                    commitAttempt.putExpectedStates(contentId, Optional.ofNullable(currentStates.get(contentId))).putGlobal(contentId, global).addPuts(KeyWithBytes.of(key, contentId, (byte) 0, put));
                    expectedGlobalStates.put(contentId, global);
                    usedContentIds.add(contentId);
                    currentStates.put(contentId, global);
                }
            }
            ImmutableCommitAttempt attempt = commitAttempt.build();
            if (!attempt.getPuts().isEmpty()) {
                heads.put(branch, databaseAdapter.commit(attempt));
            }
        }
    }
    // verify that all global-state keys (== Key + content-id) are returned (in any order)
    try (Stream<ContentId> globalKeys = databaseAdapter.globalKeys()) {
        assertThat(globalKeys).containsExactlyInAnyOrderElementsOf(expectedGlobalStates.keySet());
    }
    try (Stream<ContentIdAndBytes> allStates = databaseAdapter.globalContent(expectedGlobalStates.keySet())) {
        List<ContentIdAndBytes> all = allStates.collect(Collectors.toList());
        // verify that the global-state-log returns all keys (in any order)
        assertThat(all.stream().map(ContentIdAndBytes::getContentId).distinct()).containsExactlyInAnyOrderElementsOf(usedContentIds);
        // verify that the global-state-log returns all content-ids (in any order)
        assertThat(all.stream().map(ContentIdAndBytes::getContentId).distinct()).containsExactlyInAnyOrderElementsOf(currentStates.keySet());
        Collection<ByteString> allExpected = expectedGlobalStates.values();
        // verify that the global-state-log returns all state-values
        assertThat(all.stream().map(ContentIdAndBytes::getValue)).containsExactlyInAnyOrderElementsOf(allExpected);
    }
}

Example 97 with Hash

use of org.projectnessie.versioned.Hash in project nessie by projectnessie.

the class AbstractDatabaseAdapter method commitAttempt.

/**
 * Logic implementation of a commit-attempt.
 *
 * @param ctx technical operation-context
 * @param commitAttempt commit parameters
 * @param branchHead current HEAD of {@code branch}
 * @param newKeyLists consumer for optimistically written {@link KeyListEntity}s
 * @return optimistically written commit-log-entry
 */
protected CommitLogEntry commitAttempt(OP_CONTEXT ctx, long timeInMicros, Hash branchHead, CommitAttempt commitAttempt, Consumer<Hash> newKeyLists) throws ReferenceNotFoundException, ReferenceConflictException {
    List<String> mismatches = new ArrayList<>();
    Callable<Void> validator = commitAttempt.getValidator();
    if (validator != null) {
        try {
            validator.call();
        } catch (RuntimeException e) {
            // just propagate the RuntimeException up
            throw e;
        } catch (Exception e) {
            throw new RuntimeException(e);
        }
    }
    // verify expected global-states
    checkExpectedGlobalStates(ctx, commitAttempt, mismatches::add);
    checkForModifiedKeysBetweenExpectedAndCurrentCommit(ctx, commitAttempt, branchHead, mismatches);
    if (!mismatches.isEmpty()) {
        throw new ReferenceConflictException(String.join("\n", mismatches));
    }
    CommitLogEntry currentBranchEntry = fetchFromCommitLog(ctx, branchHead);
    int parentsPerCommit = config.getParentsPerCommit();
    List<Hash> newParents = new ArrayList<>(parentsPerCommit);
    newParents.add(branchHead);
    long commitSeq;
    if (currentBranchEntry != null) {
        List<Hash> p = currentBranchEntry.getParents();
        newParents.addAll(p.subList(0, Math.min(p.size(), parentsPerCommit - 1)));
        commitSeq = currentBranchEntry.getCommitSeq() + 1;
    } else {
        commitSeq = 1;
    }
    CommitLogEntry newBranchCommit = buildIndividualCommit(ctx, timeInMicros, newParents, commitSeq, commitAttempt.getCommitMetaSerialized(), commitAttempt.getPuts(), commitAttempt.getDeletes(), currentBranchEntry != null ? currentBranchEntry.getKeyListDistance() : 0, newKeyLists, NO_IN_MEMORY_COMMITS);
    writeIndividualCommit(ctx, newBranchCommit);
    return newBranchCommit;
}

Example 98 with Hash

use of org.projectnessie.versioned.Hash in project nessie by projectnessie.

the class PersistVersionStore method getCommits.

@Override
public Stream<Commit<METADATA, CONTENT>> getCommits(Ref ref, boolean fetchAdditionalInfo) throws ReferenceNotFoundException {
    Hash hash = refToHash(ref);
    Stream<CommitLogEntry> stream = databaseAdapter.commitLog(hash);
    BiConsumer<ImmutableCommit.Builder<METADATA, CONTENT>, CommitLogEntry> enhancer = enhancerForCommitLog(fetchAdditionalInfo);
    return stream.map(e -> {
        ImmutableCommit.Builder<METADATA, CONTENT> commit = Commit.<METADATA, CONTENT>builder().hash(e.getHash()).commitMeta(deserializeMetadata(e.getMetadata()));
        enhancer.accept(commit, e);
        return commit.build();
    });
}

Example 99 with Hash

use of org.projectnessie.versioned.Hash in project nessie by projectnessie.

the class AbstractCommitScenarios method commit.

@ParameterizedTest
@ValueSource(ints = { 1, 3, 5 })
void commit(int tablesPerCommit) throws Exception {
    BranchName branch = BranchName.of("main");
    ArrayList<Key> keys = new ArrayList<>(tablesPerCommit);
    ImmutableCommitAttempt.Builder commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("initial commit meta"));
    for (int i = 0; i < tablesPerCommit; i++) {
        Key key = Key.of("my", "table", "num" + i);
        keys.add(key);
        String cid = "id-" + i;
        WithGlobalStateContent c = WithGlobalStateContent.withGlobal("0", "initial commit content", cid);
        commit.addPuts(KeyWithBytes.of(key, ContentId.of(cid), SimpleStoreWorker.INSTANCE.getPayload(c), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c))).putGlobal(ContentId.of(cid), SimpleStoreWorker.INSTANCE.toStoreGlobalState(c)).putExpectedStates(ContentId.of(cid), Optional.empty());
    }
    Hash head = databaseAdapter.commit(commit.build());
    for (int commitNum = 0; commitNum < 3; commitNum++) {
        Map<Key, ContentAndState<ByteString>> values = databaseAdapter.values(databaseAdapter.hashOnReference(branch, Optional.empty()), keys, KeyFilterPredicate.ALLOW_ALL);
        commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("initial commit meta"));
        for (int i = 0; i < tablesPerCommit; i++) {
            String currentState = values.get(keys.get(i)).getGlobalState().toStringUtf8();
            String newGlobalState = Integer.toString(Integer.parseInt(currentState) + 1);
            String cid = "id-" + i;
            WithGlobalStateContent newContent = WithGlobalStateContent.withGlobal(newGlobalState, "branch value", cid);
            WithGlobalStateContent expectedContent = WithGlobalStateContent.withGlobal(currentState, currentState, cid);
            commit.addPuts(KeyWithBytes.of(keys.get(i), ContentId.of(cid), SimpleStoreWorker.INSTANCE.getPayload(newContent), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(newContent))).putGlobal(ContentId.of(cid), SimpleStoreWorker.INSTANCE.toStoreGlobalState(newContent)).putExpectedStates(ContentId.of(cid), Optional.of(SimpleStoreWorker.INSTANCE.toStoreGlobalState(expectedContent)));
        }
        Hash newHead = databaseAdapter.commit(commit.build());
        assertThat(newHead).isNotEqualTo(head);
        head = newHead;
    }
}