Search in sources :

Example 26 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class CommitBench method doCommit.

private void doCommit(BenchmarkParam bp, BranchName branch, List<Key> keys, Map<Key, String> contentIds) throws Exception {
    Map<Key, BaseContent> contentByKey = bp.versionStore.getValues(branch, keys);
    try {
        List<Operation<BaseContent>> operations = new ArrayList<>(bp.tablesPerCommit);
        for (int i = 0; i < bp.tablesPerCommit; i++) {
            Key key = keys.get(i);
            BaseContent value = contentByKey.get(key);
            if (value == null) {
                throw new RuntimeException("no value for key " + key + " in " + branch);
            }
            String currentState = ((WithGlobalStateContent) value).getGlobal();
            String newGlobalState = Integer.toString(Integer.parseInt(currentState) + 1);
            String contentId = contentIds.get(key);
            operations.add(Put.of(key, // hashes, because parent, "content", key are all the same.
            withGlobal(newGlobalState, "commit value " + ThreadLocalRandom.current().nextLong(), contentId), withGlobal(currentState, "foo", contentId)));
        }
        bp.versionStore.commit(branch, Optional.empty(), commitMessage("commit meta data"), operations);
        bp.success.incrementAndGet();
    } catch (ReferenceRetryFailureException e) {
        bp.retryFailures.incrementAndGet();
    } catch (ReferenceConflictException e) {
        bp.conflictsFailures.incrementAndGet();
    }
}
Also used : BaseContent(org.projectnessie.versioned.testworker.BaseContent) ReferenceConflictException(org.projectnessie.versioned.ReferenceConflictException) ArrayList(java.util.ArrayList) Operation(org.projectnessie.versioned.Operation) ReferenceRetryFailureException(org.projectnessie.versioned.ReferenceRetryFailureException) Key(org.projectnessie.versioned.Key) WithGlobalStateContent(org.projectnessie.versioned.testworker.WithGlobalStateContent)

Example 27 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class AbstractConcurrency method commitAndRecord.

private void commitAndRecord(Map<ContentId, ByteString> globalStates, Map<BranchName, Map<Key, ByteString>> onRefStates, BranchName branch, Builder commitAttempt) throws ReferenceConflictException, ReferenceNotFoundException {
    CommitAttempt c = commitAttempt.build();
    databaseAdapter.commit(c);
    globalStates.putAll(c.getGlobal());
    Map<Key, ByteString> onRef = onRefStates.computeIfAbsent(branch, b -> new ConcurrentHashMap<>());
    c.getPuts().forEach(kwb -> onRef.put(kwb.getKey(), kwb.getValue()));
}
Also used : ByteString(com.google.protobuf.ByteString) Key(org.projectnessie.versioned.Key) CommitAttempt(org.projectnessie.versioned.persist.adapter.CommitAttempt) ImmutableCommitAttempt(org.projectnessie.versioned.persist.adapter.ImmutableCommitAttempt)

Example 28 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class AbstractGlobalStates method globalStates.

/**
 * Rudimentary test for Nessie-GC related basic operations to collect all globally known keys and
 * the global-state-logs.
 */
@ParameterizedTest
@MethodSource("globalStatesParams")
void globalStates(GlobalStateParam param) throws Exception {
    List<BranchName> branches = IntStream.range(0, param.branches).mapToObj(i -> BranchName.of("globalStates-" + i)).collect(Collectors.toList());
    Map<BranchName, Hash> heads = branches.stream().collect(Collectors.toMap(b -> b, b -> catchingFunction(() -> databaseAdapter.create(b, databaseAdapter.hashOnReference(BranchName.of("main"), Optional.empty())))));
    Map<ContentId, ByteString> currentStates = new HashMap<>();
    Set<Key> keys = IntStream.range(0, param.tables).mapToObj(i -> Key.of("table", Integer.toString(i))).collect(Collectors.toSet());
    Set<ContentId> usedContentIds = new HashSet<>();
    Map<ContentId, ByteString> expectedGlobalStates = new HashMap<>();
    for (int commit = 0; commit < param.commitsPerBranch; commit++) {
        for (BranchName branch : branches) {
            ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder().commitToBranch(branch).expectedHead(Optional.of(heads.get(branch))).commitMetaSerialized(ByteString.copyFromUtf8("some commit#" + commit + " branch " + branch.getName()));
            for (Key key : keys) {
                if (param.tableCommitProbability == 1.0f || ThreadLocalRandom.current().nextDouble(0d, 1d) <= param.tableCommitProbability) {
                    String state = "state-commit-" + commit + "+" + key;
                    String value = "value-commit-" + commit + "+" + key;
                    ContentId contentId = ContentId.of(key.toString() + "-" + branch.getName());
                    ByteString put = ByteString.copyFromUtf8(value);
                    ByteString global = ByteString.copyFromUtf8(state);
                    commitAttempt.putExpectedStates(contentId, Optional.ofNullable(currentStates.get(contentId))).putGlobal(contentId, global).addPuts(KeyWithBytes.of(key, contentId, (byte) 0, put));
                    expectedGlobalStates.put(contentId, global);
                    usedContentIds.add(contentId);
                    currentStates.put(contentId, global);
                }
            }
            ImmutableCommitAttempt attempt = commitAttempt.build();
            if (!attempt.getPuts().isEmpty()) {
                heads.put(branch, databaseAdapter.commit(attempt));
            }
        }
    }
    // verify that all global-state keys (== Key + content-id) are returned (in any order)
    try (Stream<ContentId> globalKeys = databaseAdapter.globalKeys()) {
        assertThat(globalKeys).containsExactlyInAnyOrderElementsOf(expectedGlobalStates.keySet());
    }
    try (Stream<ContentIdAndBytes> allStates = databaseAdapter.globalContent(expectedGlobalStates.keySet())) {
        List<ContentIdAndBytes> all = allStates.collect(Collectors.toList());
        // verify that the global-state-log returns all keys (in any order)
        assertThat(all.stream().map(ContentIdAndBytes::getContentId).distinct()).containsExactlyInAnyOrderElementsOf(usedContentIds);
        // verify that the global-state-log returns all content-ids (in any order)
        assertThat(all.stream().map(ContentIdAndBytes::getContentId).distinct()).containsExactlyInAnyOrderElementsOf(currentStates.keySet());
        Collection<ByteString> allExpected = expectedGlobalStates.values();
        // verify that the global-state-log returns all state-values
        assertThat(all.stream().map(ContentIdAndBytes::getValue)).containsExactlyInAnyOrderElementsOf(allExpected);
    }
}
Also used : IntStream(java.util.stream.IntStream) Arrays(java.util.Arrays) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) HashMap(java.util.HashMap) HashSet(java.util.HashSet) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) Map(java.util.Map) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) DatabaseAdapter(org.projectnessie.versioned.persist.adapter.DatabaseAdapter) MethodSource(org.junit.jupiter.params.provider.MethodSource) ContentIdAndBytes(org.projectnessie.versioned.persist.adapter.ContentIdAndBytes) Hash(org.projectnessie.versioned.Hash) Collection(java.util.Collection) Set(java.util.Set) Key(org.projectnessie.versioned.Key) Collectors(java.util.stream.Collectors) KeyWithBytes(org.projectnessie.versioned.persist.adapter.KeyWithBytes) ByteString(com.google.protobuf.ByteString) Test(org.junit.jupiter.api.Test) BranchName(org.projectnessie.versioned.BranchName) ReferenceConflictException(org.projectnessie.versioned.ReferenceConflictException) List(java.util.List) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Stream(java.util.stream.Stream) ImmutableCommitAttempt(org.projectnessie.versioned.persist.adapter.ImmutableCommitAttempt) Optional(java.util.Optional) ContentId(org.projectnessie.versioned.persist.adapter.ContentId) ContentIdAndBytes(org.projectnessie.versioned.persist.adapter.ContentIdAndBytes) HashMap(java.util.HashMap) ByteString(com.google.protobuf.ByteString) ImmutableCommitAttempt(org.projectnessie.versioned.persist.adapter.ImmutableCommitAttempt) ByteString(com.google.protobuf.ByteString) Hash(org.projectnessie.versioned.Hash) ContentId(org.projectnessie.versioned.persist.adapter.ContentId) BranchName(org.projectnessie.versioned.BranchName) Key(org.projectnessie.versioned.Key) HashSet(java.util.HashSet) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) MethodSource(org.junit.jupiter.params.provider.MethodSource)

Example 29 with Key

use of org.projectnessie.versioned.Key in project nessie by projectnessie.

the class AbstractCommitScenarios method commit.

@ParameterizedTest
@ValueSource(ints = { 1, 3, 5 })
void commit(int tablesPerCommit) throws Exception {
    BranchName branch = BranchName.of("main");
    ArrayList<Key> keys = new ArrayList<>(tablesPerCommit);
    ImmutableCommitAttempt.Builder commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("initial commit meta"));
    for (int i = 0; i < tablesPerCommit; i++) {
        Key key = Key.of("my", "table", "num" + i);
        keys.add(key);
        String cid = "id-" + i;
        WithGlobalStateContent c = WithGlobalStateContent.withGlobal("0", "initial commit content", cid);
        commit.addPuts(KeyWithBytes.of(key, ContentId.of(cid), SimpleStoreWorker.INSTANCE.getPayload(c), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c))).putGlobal(ContentId.of(cid), SimpleStoreWorker.INSTANCE.toStoreGlobalState(c)).putExpectedStates(ContentId.of(cid), Optional.empty());
    }
    Hash head = databaseAdapter.commit(commit.build());
    for (int commitNum = 0; commitNum < 3; commitNum++) {
        Map<Key, ContentAndState<ByteString>> values = databaseAdapter.values(databaseAdapter.hashOnReference(branch, Optional.empty()), keys, KeyFilterPredicate.ALLOW_ALL);
        commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("initial commit meta"));
        for (int i = 0; i < tablesPerCommit; i++) {
            String currentState = values.get(keys.get(i)).getGlobalState().toStringUtf8();
            String newGlobalState = Integer.toString(Integer.parseInt(currentState) + 1);
            String cid = "id-" + i;
            WithGlobalStateContent newContent = WithGlobalStateContent.withGlobal(newGlobalState, "branch value", cid);
            WithGlobalStateContent expectedContent = WithGlobalStateContent.withGlobal(currentState, currentState, cid);
            commit.addPuts(KeyWithBytes.of(keys.get(i), ContentId.of(cid), SimpleStoreWorker.INSTANCE.getPayload(newContent), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(newContent))).putGlobal(ContentId.of(cid), SimpleStoreWorker.INSTANCE.toStoreGlobalState(newContent)).putExpectedStates(ContentId.of(cid), Optional.of(SimpleStoreWorker.INSTANCE.toStoreGlobalState(expectedContent)));
        }
        Hash newHead = databaseAdapter.commit(commit.build());
        assertThat(newHead).isNotEqualTo(head);
        head = newHead;
    }
}
Also used : ImmutableCommitAttempt(org.projectnessie.versioned.persist.adapter.ImmutableCommitAttempt) ContentAndState(org.projectnessie.versioned.persist.adapter.ContentAndState) ArrayList(java.util.ArrayList) BranchName(org.projectnessie.versioned.BranchName) ByteString(com.google.protobuf.ByteString) Hash(org.projectnessie.versioned.Hash) Key(org.projectnessie.versioned.Key) WithGlobalStateContent(org.projectnessie.versioned.testworker.WithGlobalStateContent) ValueSource(org.junit.jupiter.params.provider.ValueSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

Key (org.projectnessie.versioned.Key)29 BranchName (org.projectnessie.versioned.BranchName)21 Hash (org.projectnessie.versioned.Hash)21 ByteString (com.google.protobuf.ByteString)14 List (java.util.List)12 Collectors (java.util.stream.Collectors)12 Stream (java.util.stream.Stream)12 ReferenceConflictException (org.projectnessie.versioned.ReferenceConflictException)12 ArrayList (java.util.ArrayList)11 Map (java.util.Map)11 Optional (java.util.Optional)11 ReferenceNotFoundException (org.projectnessie.versioned.ReferenceNotFoundException)11 Test (org.junit.jupiter.api.Test)10 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)10 ContentId (org.projectnessie.versioned.persist.adapter.ContentId)10 IntStream (java.util.stream.IntStream)9 KeyWithBytes (org.projectnessie.versioned.persist.adapter.KeyWithBytes)9 Collections (java.util.Collections)8 ContentAndState (org.projectnessie.versioned.persist.adapter.ContentAndState)8 DatabaseAdapter (org.projectnessie.versioned.persist.adapter.DatabaseAdapter)8