Examples with Key org.projectnessie.versioned.Key used on opensource projects

Example 26 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class CommitBench method doCommit.

private void doCommit(BenchmarkParam bp, BranchName branch, List<Key> keys, Map<Key, String> contentIds) throws Exception {
    Map<Key, BaseContent> contentByKey = bp.versionStore.getValues(branch, keys);
    try {
        List<Operation<BaseContent>> operations = new ArrayList<>(bp.tablesPerCommit);
        for (int i = 0; i < bp.tablesPerCommit; i++) {
            Key key = keys.get(i);
            BaseContent value = contentByKey.get(key);
            if (value == null) {
                throw new RuntimeException("no value for key " + key + " in " + branch);
            }
            String currentState = ((WithGlobalStateContent) value).getGlobal();
            String newGlobalState = Integer.toString(Integer.parseInt(currentState) + 1);
            String contentId = contentIds.get(key);
            operations.add(Put.of(key, // hashes, because parent, "content", key are all the same.
            withGlobal(newGlobalState, "commit value " + ThreadLocalRandom.current().nextLong(), contentId), withGlobal(currentState, "foo", contentId)));
        }
        bp.versionStore.commit(branch, Optional.empty(), commitMessage("commit meta data"), operations);
        bp.success.incrementAndGet();
    } catch (ReferenceRetryFailureException e) {
        bp.retryFailures.incrementAndGet();
    } catch (ReferenceConflictException e) {
        bp.conflictsFailures.incrementAndGet();
    }
}

Example 27 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class AbstractConcurrency method commitAndRecord.

private void commitAndRecord(Map<ContentId, ByteString> globalStates, Map<BranchName, Map<Key, ByteString>> onRefStates, BranchName branch, Builder commitAttempt) throws ReferenceConflictException, ReferenceNotFoundException {
    CommitAttempt c = commitAttempt.build();
    databaseAdapter.commit(c);
    globalStates.putAll(c.getGlobal());
    Map<Key, ByteString> onRef = onRefStates.computeIfAbsent(branch, b -> new ConcurrentHashMap<>());
    c.getPuts().forEach(kwb -> onRef.put(kwb.getKey(), kwb.getValue()));
}

Example 28 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class AbstractGlobalStates method globalStates.

/**
 * Rudimentary test for Nessie-GC related basic operations to collect all globally known keys and
 * the global-state-logs.
 */
@ParameterizedTest
@MethodSource("globalStatesParams")
void globalStates(GlobalStateParam param) throws Exception {
    List<BranchName> branches = IntStream.range(0, param.branches).mapToObj(i -> BranchName.of("globalStates-" + i)).collect(Collectors.toList());
    Map<BranchName, Hash> heads = branches.stream().collect(Collectors.toMap(b -> b, b -> catchingFunction(() -> databaseAdapter.create(b, databaseAdapter.hashOnReference(BranchName.of("main"), Optional.empty())))));
    Map<ContentId, ByteString> currentStates = new HashMap<>();
    Set<Key> keys = IntStream.range(0, param.tables).mapToObj(i -> Key.of("table", Integer.toString(i))).collect(Collectors.toSet());
    Set<ContentId> usedContentIds = new HashSet<>();
    Map<ContentId, ByteString> expectedGlobalStates = new HashMap<>();
    for (int commit = 0; commit < param.commitsPerBranch; commit++) {
        for (BranchName branch : branches) {
            ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder().commitToBranch(branch).expectedHead(Optional.of(heads.get(branch))).commitMetaSerialized(ByteString.copyFromUtf8("some commit#" + commit + " branch " + branch.getName()));
            for (Key key : keys) {
                if (param.tableCommitProbability == 1.0f || ThreadLocalRandom.current().nextDouble(0d, 1d) <= param.tableCommitProbability) {
                    String state = "state-commit-" + commit + "+" + key;
                    String value = "value-commit-" + commit + "+" + key;
                    ContentId contentId = ContentId.of(key.toString() + "-" + branch.getName());
                    ByteString put = ByteString.copyFromUtf8(value);
                    ByteString global = ByteString.copyFromUtf8(state);
                    commitAttempt.putExpectedStates(contentId, Optional.ofNullable(currentStates.get(contentId))).putGlobal(contentId, global).addPuts(KeyWithBytes.of(key, contentId, (byte) 0, put));
                    expectedGlobalStates.put(contentId, global);
                    usedContentIds.add(contentId);
                    currentStates.put(contentId, global);
                }
            }
            ImmutableCommitAttempt attempt = commitAttempt.build();
            if (!attempt.getPuts().isEmpty()) {
                heads.put(branch, databaseAdapter.commit(attempt));
            }
        }
    }
    // verify that all global-state keys (== Key + content-id) are returned (in any order)
    try (Stream<ContentId> globalKeys = databaseAdapter.globalKeys()) {
        assertThat(globalKeys).containsExactlyInAnyOrderElementsOf(expectedGlobalStates.keySet());
    }
    try (Stream<ContentIdAndBytes> allStates = databaseAdapter.globalContent(expectedGlobalStates.keySet())) {
        List<ContentIdAndBytes> all = allStates.collect(Collectors.toList());
        // verify that the global-state-log returns all keys (in any order)
        assertThat(all.stream().map(ContentIdAndBytes::getContentId).distinct()).containsExactlyInAnyOrderElementsOf(usedContentIds);
        // verify that the global-state-log returns all content-ids (in any order)
        assertThat(all.stream().map(ContentIdAndBytes::getContentId).distinct()).containsExactlyInAnyOrderElementsOf(currentStates.keySet());
        Collection<ByteString> allExpected = expectedGlobalStates.values();
        // verify that the global-state-log returns all state-values
        assertThat(all.stream().map(ContentIdAndBytes::getValue)).containsExactlyInAnyOrderElementsOf(allExpected);
    }
}

Example 29 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class AbstractCommitScenarios method commit.

@ParameterizedTest
@ValueSource(ints = { 1, 3, 5 })
void commit(int tablesPerCommit) throws Exception {
    BranchName branch = BranchName.of("main");
    ArrayList<Key> keys = new ArrayList<>(tablesPerCommit);
    ImmutableCommitAttempt.Builder commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("initial commit meta"));
    for (int i = 0; i < tablesPerCommit; i++) {
        Key key = Key.of("my", "table", "num" + i);
        keys.add(key);
        String cid = "id-" + i;
        WithGlobalStateContent c = WithGlobalStateContent.withGlobal("0", "initial commit content", cid);
        commit.addPuts(KeyWithBytes.of(key, ContentId.of(cid), SimpleStoreWorker.INSTANCE.getPayload(c), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c))).putGlobal(ContentId.of(cid), SimpleStoreWorker.INSTANCE.toStoreGlobalState(c)).putExpectedStates(ContentId.of(cid), Optional.empty());
    }
    Hash head = databaseAdapter.commit(commit.build());
    for (int commitNum = 0; commitNum < 3; commitNum++) {
        Map<Key, ContentAndState<ByteString>> values = databaseAdapter.values(databaseAdapter.hashOnReference(branch, Optional.empty()), keys, KeyFilterPredicate.ALLOW_ALL);
        commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("initial commit meta"));
        for (int i = 0; i < tablesPerCommit; i++) {
            String currentState = values.get(keys.get(i)).getGlobalState().toStringUtf8();
            String newGlobalState = Integer.toString(Integer.parseInt(currentState) + 1);
            String cid = "id-" + i;
            WithGlobalStateContent newContent = WithGlobalStateContent.withGlobal(newGlobalState, "branch value", cid);
            WithGlobalStateContent expectedContent = WithGlobalStateContent.withGlobal(currentState, currentState, cid);
            commit.addPuts(KeyWithBytes.of(keys.get(i), ContentId.of(cid), SimpleStoreWorker.INSTANCE.getPayload(newContent), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(newContent))).putGlobal(ContentId.of(cid), SimpleStoreWorker.INSTANCE.toStoreGlobalState(newContent)).putExpectedStates(ContentId.of(cid), Optional.of(SimpleStoreWorker.INSTANCE.toStoreGlobalState(expectedContent)));
        }
        Hash newHead = databaseAdapter.commit(commit.build());
        assertThat(newHead).isNotEqualTo(head);
        head = newHead;
    }
}