Examples with Key org.projectnessie.versioned.Key used on opensource projects

Example 21 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class AbstractManyCommits method manyCommits.

@ParameterizedTest
@ValueSource(ints = { 0, 1, 19, 20, 21, 39, 40, 41, 49, 50, 51, 255, 256, 257, 500 })
// Note: 1000 commits is quite the max that in-JVM H2 database can handle
void manyCommits(int numCommits) throws Exception {
    BranchName branch = BranchName.of("manyCommits-" + numCommits);
    databaseAdapter.create(branch, databaseAdapter.hashOnReference(BranchName.of("main"), Optional.empty()));
    Hash[] commits = new Hash[numCommits];
    ContentId fixed = ContentId.of("FIXED");
    for (int i = 0; i < numCommits; i++) {
        Key key = Key.of("many", "commits", Integer.toString(numCommits));
        WithGlobalStateContent c = WithGlobalStateContent.withGlobal("state for #" + i + " of " + numCommits, "value for #" + i + " of " + numCommits, fixed.getId());
        byte payload = SimpleStoreWorker.INSTANCE.getPayload(c);
        ImmutableCommitAttempt.Builder commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("commit #" + i + " of " + numCommits)).addPuts(KeyWithBytes.of(key, fixed, payload, SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c))).putGlobal(fixed, SimpleStoreWorker.INSTANCE.toStoreGlobalState(c));
        if (i > 0) {
            WithGlobalStateContent expected = WithGlobalStateContent.withGlobal("state for #" + (i - 1) + " of " + numCommits, "value for #" + (i - 1) + " of " + numCommits, fixed.getId());
            commit.putExpectedStates(fixed, Optional.of(SimpleStoreWorker.INSTANCE.toStoreGlobalState(expected)));
        }
        Hash hash = databaseAdapter.commit(commit.build());
        commits[i] = hash;
        try (Stream<ContentIdAndBytes> globals = databaseAdapter.globalContent(Collections.singleton(fixed))) {
            WithGlobalStateContent expected = WithGlobalStateContent.withGlobal("state for #" + i + " of " + numCommits, "value for #" + i + " of " + numCommits, fixed.getId());
            assertThat(globals).containsExactly(ContentIdAndBytes.of(fixed, SimpleStoreWorker.INSTANCE.toStoreGlobalState(expected)));
        }
    }
    try (Stream<CommitLogEntry> log = databaseAdapter.commitLog(databaseAdapter.hashOnReference(branch, Optional.empty()))) {
        assertThat(log.count()).isEqualTo(numCommits);
    }
    ExecutorService executor = Executors.newFixedThreadPool(Math.max(4, Runtime.getRuntime().availableProcessors()));
    try {
        CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(IntStream.range(0, numCommits).mapToObj(i -> (Runnable) () -> verify(i, numCommits, branch, commits[i], fixed)).map(r -> CompletableFuture.runAsync(r, executor)).toArray((IntFunction<CompletableFuture<?>[]>) CompletableFuture[]::new));
        combinedFuture.get();
    } finally {
        executor.shutdown();
    }
    databaseAdapter.delete(branch, Optional.empty());
}

Example 22 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class AbstractManyKeys method manyKeys.

@ParameterizedTest
@MethodSource("manyKeysParams")
void manyKeys(ManyKeysParams params) throws Exception {
    BranchName main = BranchName.of("main");
    List<ImmutableCommitAttempt.Builder> commits = IntStream.range(0, params.commits).mapToObj(i -> ImmutableCommitAttempt.builder().commitMetaSerialized(ByteString.copyFromUtf8("commit #" + i)).commitToBranch(main)).collect(Collectors.toList());
    AtomicInteger commitDist = new AtomicInteger();
    Set<Key> allKeys = new HashSet<>();
    IntStream.range(0, params.keys).mapToObj(i -> {
        Key key = Key.of("some", Integer.toString(i), "long", "key", "value", "foobarbazfoobarbazfoobarbazfoobarbazfoobarbazfoobarbaz");
        allKeys.add(key);
        return KeyWithBytes.of(key, ContentId.of("cid-" + i), (byte) 0, ByteString.copyFromUtf8("value " + i));
    }).forEach(kb -> commits.get(commitDist.incrementAndGet() % params.commits).addPuts(kb));
    for (ImmutableCommitAttempt.Builder commit : commits) {
        databaseAdapter.commit(commit.build());
    }
    Hash mainHead = databaseAdapter.hashOnReference(main, Optional.empty());
    try (Stream<KeyListEntry> keys = databaseAdapter.keys(mainHead, KeyFilterPredicate.ALLOW_ALL)) {
        List<Key> fetchedKeys = keys.map(KeyListEntry::getKey).collect(Collectors.toList());
        // containsExactlyInAnyOrderElementsOf() is quite expensive and slow with Key's
        // implementation of 'Key.equals()' since it uses a collator.
        List<String> fetchedKeysStrings = fetchedKeys.stream().map(Key::toString).collect(Collectors.toList());
        List<String> allKeysStrings = allKeys.stream().map(Key::toString).collect(Collectors.toList());
        assertThat(fetchedKeysStrings).hasSize(allKeysStrings.size()).containsExactlyInAnyOrderElementsOf(allKeysStrings);
    }
}

Example 23 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class AbstractSingleBranch method singleBranchManyUsers.

/**
 * Use case simulation matrix: single branch, multiple users, each or all user updating a separate
 * or single table.
 */
@ParameterizedTest
@MethodSource("singleBranchManyUsersCases")
void singleBranchManyUsers(SingleBranchParam param) throws Exception {
    BranchName branch = BranchName.of(param.branchName);
    int numUsers = 5;
    int numCommits = 50;
    Hash[] hashesKnownByUser = new Hash[numUsers];
    Hash createHash = store().create(branch, Optional.empty());
    Arrays.fill(hashesKnownByUser, createHash);
    List<CommitMessage> expectedValues = new ArrayList<>();
    Map<Key, String> previousState = new HashMap<>();
    for (int commitNum = 0; commitNum < numCommits; commitNum++) {
        for (int user = 0; user < numUsers; user++) {
            Hash hashKnownByUser = hashesKnownByUser[user];
            CommitMessage msg = commitMessage(String.format("user %03d/commit %03d", user, commitNum));
            expectedValues.add(msg);
            Key key = Key.of(param.tableNameGen.apply(user));
            String contentId = param.contentIdGen.apply(user);
            Operation<BaseContent> put;
            if (param.globalState) {
                String state = String.format("%03d_%03d", user, commitNum);
                if (previousState.containsKey(key)) {
                    put = Put.of(key, withGlobal(state, "data_file", contentId), withGlobal(previousState.get(key), "foo", contentId));
                } else {
                    put = Put.of(key, withGlobal(state, "data_file", contentId));
                }
                previousState.put(key, state);
            } else {
                BaseContent value = newOnRef(String.format("data_file_%03d_%03d", user, commitNum));
                put = Put.of(key, value);
            }
            Hash commitHash;
            List<Operation<BaseContent>> ops = ImmutableList.of(put);
            try {
                commitHash = store().commit(branch, Optional.of(hashKnownByUser), msg, ops);
            } catch (ReferenceConflictException inconsistentValueException) {
                if (param.allowInconsistentValueException) {
                    hashKnownByUser = store().hashOnReference(branch, Optional.empty());
                    commitHash = store().commit(branch, Optional.of(hashKnownByUser), msg, ops);
                } else {
                    throw inconsistentValueException;
                }
            }
            assertNotEquals(hashKnownByUser, commitHash);
            hashesKnownByUser[user] = commitHash;
        }
    }
    // Verify that all commits are there and that the order of the commits is correct
    List<CommitMessage> committedValues = commitsList(branch, s -> s.map(Commit::getCommitMeta), false);
    Collections.reverse(expectedValues);
    assertEquals(expectedValues, committedValues);
}

Example 24 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class AbstractContents method recreateTable.

@Test
void recreateTable() throws Exception {
    BranchName branch = BranchName.of("recreateTable-main");
    Key key = Key.of("recreateTable");
    store().create(branch, Optional.empty());
    // commit just something to have a "real" common ancestor and not "beginning of time", which
    // means no-common-ancestor
    BaseContent initialState = newWithGlobal("initial-state", "value");
    Hash ancestor = store().commit(branch, Optional.empty(), commitMessage("create table"), singletonList(Put.of(key, initialState)));
    assertThat(store().getValue(branch, key)).isEqualTo(initialState);
    assertThat(store().getValue(ancestor, key)).isEqualTo(initialState);
    Hash delete = store().commit(branch, Optional.empty(), commitMessage("drop table"), ImmutableList.of(Delete.of(key)));
    assertThat(store().getValue(branch, key)).isNull();
    assertThat(store().getValue(delete, key)).isNull();
    BaseContent recreateState = newWithGlobal("recreate-state", "value");
    Hash recreate = store().commit(branch, Optional.empty(), commitMessage("drop table"), ImmutableList.of(Put.of(key, recreateState)));
    assertThat(store().getValue(branch, key)).isEqualTo(recreateState);
    assertThat(store().getValue(recreate, key)).isEqualTo(recreateState);
}

Example 25 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class AbstractDatabaseAdapter method mergeAttempt.

/**
 * Logic implementation of a merge-attempt.
 *
 * @param ctx technical operation context
 * @param from merge-from commit
 * @param toBranch merge-into reference with expected hash of HEAD
 * @param expectedHead if present, {@code toBranch}'s current HEAD must be equal to this value
 * @param toHead current HEAD of {@code toBranch}
 * @param branchCommits consumer for the individual commits to merge
 * @param newKeyLists consumer for optimistically written {@link KeyListEntity}s
 * @param rewriteMetadata function to rewrite the commit-metadata for copied commits
 * @return hash of the last commit-log-entry written to {@code toBranch}
 */
protected Hash mergeAttempt(OP_CONTEXT ctx, long timeInMicros, Hash from, BranchName toBranch, Optional<Hash> expectedHead, Hash toHead, Consumer<Hash> branchCommits, Consumer<Hash> newKeyLists, Function<ByteString, ByteString> rewriteMetadata) throws ReferenceNotFoundException, ReferenceConflictException {
    validateHashExists(ctx, from);
    // 1. ensure 'expectedHash' is a parent of HEAD-of-'toBranch'
    hashOnRef(ctx, toBranch, expectedHead, toHead);
    // 2. find nearest common-ancestor between 'from' + 'fromHash'
    Hash commonAncestor = findCommonAncestor(ctx, from, toBranch, toHead);
    // 3. Collect commit-log-entries
    List<CommitLogEntry> toEntriesReverseChronological = takeUntilExcludeLast(readCommitLogStream(ctx, toHead), e -> e.getHash().equals(commonAncestor)).collect(Collectors.toList());
    Collections.reverse(toEntriesReverseChronological);
    List<CommitLogEntry> commitsToMergeChronological = takeUntilExcludeLast(readCommitLogStream(ctx, from), e -> e.getHash().equals(commonAncestor)).collect(Collectors.toList());
    if (commitsToMergeChronological.isEmpty()) {
        // Nothing to merge, shortcut
        throw new IllegalArgumentException(String.format("No hashes to merge from '%s' onto '%s' @ '%s'.", from.asString(), toBranch.getName(), toHead));
    }
    // 4. Collect modified keys.
    Set<Key> keysTouchedOnTarget = collectModifiedKeys(toEntriesReverseChronological);
    // 5. check for key-collisions
    checkForKeyCollisions(ctx, toHead, keysTouchedOnTarget, commitsToMergeChronological);
    // (no need to verify the global states during a transplant)
    // 6. re-apply commits in 'sequenceToTransplant' onto 'targetBranch'
    toHead = copyCommits(ctx, timeInMicros, toHead, commitsToMergeChronological, newKeyLists, rewriteMetadata);
    // 7. Write commits
    commitsToMergeChronological.stream().map(CommitLogEntry::getHash).forEach(branchCommits);
    writeMultipleCommits(ctx, commitsToMergeChronological);
    return toHead;
}