Examples with BranchName org.projectnessie.versioned.BranchName used on opensource projects

Example 1 with BranchName

use of org.projectnessie.versioned.BranchName in project nessie by projectnessie.

the class AbstractCommitScenarios method commitWithValidation.

@Test
void commitWithValidation() throws Exception {
    BranchName branch = BranchName.of("main");
    Key key = Key.of("my", "table0");
    Hash branchHead = databaseAdapter.namedRef(branch.getName(), GetNamedRefsParams.DEFAULT).getHash();
    String cid = "cid-0";
    RuntimeException exception = new ArithmeticException("Whatever");
    assertThatThrownBy(() -> doCommitWithValidation(branch, cid, key, () -> {
        // do some operations here
        databaseAdapter.globalContent(ContentId.of(cid));
        assertThat(databaseAdapter.values(branchHead, Collections.singleton(key), KeyFilterPredicate.ALLOW_ALL)).isEmpty();
        // let the custom commit-validation fail
        throw exception;
    })).isSameAs(exception);
    assertThat(databaseAdapter.namedRef(branch.getName(), GetNamedRefsParams.DEFAULT).getHash()).isEqualTo(branchHead);
    assertThat(databaseAdapter.globalContent(ContentId.of(cid))).isEmpty();
}

Example 2 with BranchName

use of org.projectnessie.versioned.BranchName in project nessie by projectnessie.

the class AbstractCommitScenarios method commitRenameTable.

/**
 * Test whether a "table rename operation" works as expected.
 *
 * <p>A "table rename" is effectively a remove-operation plus a put-operation with a different key
 * but the same content-id.
 *
 * <p>Parameterized to force operations to cross/not-cross the number of commits between "full key
 * lists" and whether global-state shall be used.
 */
@ParameterizedTest
@MethodSource("commitRenameTableParams")
void commitRenameTable(RenameTable param) throws Exception {
    BranchName branch = BranchName.of("main");
    Key dummyKey = Key.of("dummy");
    Key oldKey = Key.of("hello", "table");
    Key newKey = Key.of("new", "name");
    ContentId contentId = ContentId.of("id-42");
    IntFunction<Hash> performDummyCommit = i -> {
        try {
            return databaseAdapter.commit(ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("dummy commit meta " + i)).addUnchanged(dummyKey).build());
        } catch (ReferenceNotFoundException | ReferenceConflictException e) {
            throw new RuntimeException(e);
        }
    };
    List<Hash> beforeInitial = IntStream.range(0, param.setupCommits).mapToObj(performDummyCommit).collect(Collectors.toList());
    ImmutableCommitAttempt.Builder commit;
    BaseContent initialContent;
    BaseContent renamContent;
    if (param.globalState) {
        initialContent = WithGlobalStateContent.newWithGlobal("0", "initial commit content");
        renamContent = WithGlobalStateContent.withGlobal("0", "rename commit content", initialContent.getId());
    } else {
        initialContent = OnRefOnly.newOnRef("initial commit content");
        renamContent = OnRefOnly.onRef("rename commit content", initialContent.getId());
    }
    byte payload = SimpleStoreWorker.INSTANCE.getPayload(initialContent);
    commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("initial commit meta")).addPuts(KeyWithBytes.of(oldKey, contentId, payload, SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(initialContent)));
    if (param.globalState) {
        commit.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(initialContent)).putExpectedStates(contentId, Optional.empty());
    }
    Hash hashInitial = databaseAdapter.commit(commit.build());
    List<Hash> beforeRename = IntStream.range(0, param.afterInitialCommits).mapToObj(performDummyCommit).collect(Collectors.toList());
    commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("rename table")).addDeletes(oldKey).addPuts(KeyWithBytes.of(newKey, contentId, payload, SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(renamContent)));
    if (param.globalState) {
        commit.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(renamContent)).putExpectedStates(contentId, Optional.of(SimpleStoreWorker.INSTANCE.toStoreGlobalState(initialContent)));
    }
    Hash hashRename = databaseAdapter.commit(commit.build());
    List<Hash> beforeDelete = IntStream.range(0, param.afterRenameCommits).mapToObj(performDummyCommit).collect(Collectors.toList());
    commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("delete table")).addDeletes(newKey);
    if (param.globalState) {
        commit.putGlobal(contentId, ByteString.copyFromUtf8("0")).putExpectedStates(contentId, Optional.of(ByteString.copyFromUtf8("0")));
    }
    Hash hashDelete = databaseAdapter.commit(commit.build());
    List<Hash> afterDelete = IntStream.range(0, param.afterDeleteCommits).mapToObj(performDummyCommit).collect(Collectors.toList());
    int expectedCommitCount = 1;
    // Verify that the commits before the initial put return _no_ keys
    expectedCommitCount = renameCommitVerify(beforeInitial.stream(), expectedCommitCount, keys -> assertThat(keys).isEmpty());
    // Verify that the commits since the initial put and before the rename-operation return the
    // _old_ key
    expectedCommitCount = renameCommitVerify(Stream.concat(Stream.of(hashInitial), beforeRename.stream()), expectedCommitCount, keys -> assertThat(keys).containsExactly(KeyListEntry.of(oldKey, contentId, payload, hashInitial)));
    // Verify that the commits since the rename-operation and before the delete-operation return the
    // _new_ key
    expectedCommitCount = renameCommitVerify(Stream.concat(Stream.of(hashRename), beforeDelete.stream()), expectedCommitCount, keys -> assertThat(keys).containsExactly(KeyListEntry.of(newKey, contentId, payload, hashRename)));
    // Verify that the commits since the delete-operation return _no_ keys
    expectedCommitCount = renameCommitVerify(Stream.concat(Stream.of(hashDelete), afterDelete.stream()), expectedCommitCount, keys -> assertThat(keys).isEmpty());
    assertThat(expectedCommitCount - 1).isEqualTo(param.setupCommits + 1 + param.afterInitialCommits + 1 + param.afterRenameCommits + 1 + param.afterDeleteCommits);
}

Example 3 with BranchName

use of org.projectnessie.versioned.BranchName in project nessie by projectnessie.

the class AbstractCompactGlobalLog method compactGlobalLog.

@ParameterizedTest
@MethodSource("compactGlobalLog")
public void compactGlobalLog(int commits, int numContentIds) throws Exception {
    RepoMaintenanceParams repoMaintenanceParams = RepoMaintenanceParams.builder().globalLogCompactionParams(GlobalLogCompactionParams.builder().noCompactionWhenCompactedWithin(20).noCompactionUpToLength(20).build()).build();
    Map<String, Map<String, String>> statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    ContentIdEmitter contentIdEmitter = new ContentIdEmitter(numContentIds);
    // If there's no statistics entry for global-log-compaction, then it's not a non-transactional
    // database adapter, so no global-log to compact.
    assumeThat(statistics).containsKey("compactGlobalLog");
    // An "empty repository" should not require compaction
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    BranchName branch = BranchName.of("compactGlobalLog");
    Map<ContentId, ByteString> currentGlobal = new HashMap<>();
    databaseAdapter.create(branch, databaseAdapter.noAncestorHash());
    Runnable verify = () -> {
        try (Stream<ContentIdAndBytes> globals = databaseAdapter.globalContent(contentIdEmitter.allAsSet())) {
            assertThat(globals).hasSize(contentIdEmitter.size()).allSatisfy(cb -> assertThat(currentGlobal.get(cb.getContentId())).isEqualTo(cb.getValue()));
        }
    };
    for (int i = 0; i < commits; i++) {
        commitForGlobalLogCompaction(commits, contentIdEmitter, branch, currentGlobal, i);
    }
    // Verify
    verify.run();
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "true").containsEntry("entries.puts", Long.toString(commits)).containsEntry("entries.uniquePuts", Long.toString(contentIdEmitter.size())).containsEntry("entries.read", Long.toString(commits + 2)).containsEntry("entries.read.total", Long.toString(commits + 2));
    // Verify again
    verify.run();
    // Compact again, compaction must not run, because there is at least one compacted
    // global-log-entry in the first page (above only added 5 "uncompacted" global-log-entries).
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    // Add some more commits, but not enough to trigger compaction
    int additionalCommits = 5;
    for (int i = 0; i < additionalCommits; i++) {
        commitForGlobalLogCompaction(commits + additionalCommits, contentIdEmitter, branch, currentGlobal, i + commits);
    }
    // Compact again, compaction must not run, because there is at least one compacted
    // global-log-entry in the first page (above only added 5 "uncompacted" global-log-entries).
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    // Add some more commits, enough to trigger compaction again
    int additionalCommits2 = 15;
    for (int i = 0; i < additionalCommits2; i++) {
        commitForGlobalLogCompaction(commits + additionalCommits + additionalCommits2, contentIdEmitter, branch, currentGlobal, i + commits + additionalCommits);
    }
    // Compact again, compaction must run, because there is no compacted global-log-entry in the
    // first page of the global log.
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "true").containsEntry("entries.uniquePuts", Long.toString(contentIdEmitter.size()));
    // Verify again
    verify.run();
}

Example 4 with BranchName

use of org.projectnessie.versioned.BranchName in project nessie by projectnessie.

the class AbstractConcurrency method concurrency.

@ParameterizedTest
@MethodSource("concurrencyVariations")
void concurrency(Variation variation) throws Exception {
    new ArrayList<>(globalRegistry.getRegistries()).forEach(globalRegistry::remove);
    globalRegistry.add(new SimpleMeterRegistry());
    ExecutorService executor = Executors.newFixedThreadPool(variation.threads);
    AtomicInteger commitsOK = new AtomicInteger();
    AtomicInteger retryFailures = new AtomicInteger();
    AtomicBoolean stopFlag = new AtomicBoolean();
    List<Runnable> tasks = new ArrayList<>(variation.threads);
    Map<Key, ContentId> keyToContentId = new HashMap<>();
    Map<ContentId, ByteString> globalStates = new ConcurrentHashMap<>();
    Map<BranchName, Map<Key, ByteString>> onRefStates = new ConcurrentHashMap<>();
    try {
        CountDownLatch startLatch = new CountDownLatch(1);
        Map<BranchName, Set<Key>> keysPerBranch = new HashMap<>();
        for (int i = 0; i < variation.threads; i++) {
            BranchName branch = BranchName.of("concurrency-" + ((variation.singleBranch ? "shared" : i)));
            List<Key> keys = new ArrayList<>(variation.tables);
            for (int k = 0; k < variation.tables; k++) {
                String variationKey = variation.sharedKeys ? "shared" : Integer.toString(i);
                Key key = Key.of("some", "key", variationKey, "table-" + k);
                keys.add(key);
                keyToContentId.put(key, ContentId.of(String.format("%s-table-%d", variationKey, k)));
                keysPerBranch.computeIfAbsent(branch, x -> new HashSet<>()).add(key);
            }
            tasks.add(() -> {
                try {
                    assertThat(startLatch.await(2, TimeUnit.SECONDS)).isTrue();
                    for (int commit = 0; ; commit++) {
                        if (stopFlag.get()) {
                            break;
                        }
                        List<ByteString> currentStates = databaseAdapter.values(databaseAdapter.hashOnReference(branch, Optional.empty()), keys, KeyFilterPredicate.ALLOW_ALL).values().stream().map(ContentAndState::getGlobalState).collect(Collectors.toList());
                        ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder();
                        for (int ki = 0; ki < keys.size(); ki++) {
                            Key key = keys.get(ki);
                            ContentId contentId = keyToContentId.get(key);
                            String newGlobal = Integer.toString(Integer.parseInt(currentStates.get(ki).toStringUtf8()) + 1);
                            WithGlobalStateContent c = WithGlobalStateContent.withGlobal(newGlobal, "", contentId.getId());
                            commitAttempt.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(c));
                            if (!variation.sharedKeys) {
                                commitAttempt.putExpectedStates(contentId, Optional.of(currentStates.get(ki)));
                            }
                            commitAttempt.addPuts(KeyWithBytes.of(keys.get(ki), contentId, SimpleStoreWorker.INSTANCE.getPayload(c), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c)));
                        }
                        try {
                            commitAttempt.commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("commit #" + commit + " to " + branch.getName() + " something " + ThreadLocalRandom.current().nextLong()));
                            commitAndRecord(globalStates, onRefStates, branch, commitAttempt);
                            commitsOK.incrementAndGet();
                        } catch (ReferenceRetryFailureException retry) {
                            retryFailures.incrementAndGet();
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    throw new RuntimeException(e);
                }
            });
        }
        for (Entry<BranchName, Set<Key>> branchKeys : keysPerBranch.entrySet()) {
            BranchName branch = branchKeys.getKey();
            databaseAdapter.create(branch, databaseAdapter.hashOnReference(BranchName.of("main"), Optional.empty()));
            ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder().commitToBranch(branchKeys.getKey()).commitMetaSerialized(ByteString.copyFromUtf8("initial commit for " + branch.getName()));
            for (Key k : branchKeys.getValue()) {
                ContentId contentId = keyToContentId.get(k);
                WithGlobalStateContent c = WithGlobalStateContent.withGlobal("0", "", contentId.getId());
                commitAttempt.addPuts(KeyWithBytes.of(k, contentId, SimpleStoreWorker.INSTANCE.getPayload(c), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c)));
                commitAttempt.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(c));
            }
            commitAndRecord(globalStates, onRefStates, branch, commitAttempt);
        }
        // Submit all 'Runnable's as 'CompletableFuture's and construct a combined 'CompletableFuture'
        // that we can wait for.
        CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(tasks.stream().map(r -> CompletableFuture.runAsync(r, executor)).toArray((IntFunction<CompletableFuture<?>[]>) CompletableFuture[]::new));
        startLatch.countDown();
        Thread.sleep(2_000);
        stopFlag.set(true);
        // 30 seconds is long, but necessary to let transactional databases detect deadlocks, which
        // cause Nessie-commit-retries.
        combinedFuture.get(30, TimeUnit.SECONDS);
        for (Entry<BranchName, Set<Key>> branchKeys : keysPerBranch.entrySet()) {
            BranchName branch = branchKeys.getKey();
            Hash hash = databaseAdapter.hashOnReference(branch, Optional.empty());
            ArrayList<Key> keys = new ArrayList<>(branchKeys.getValue());
            // Note: only fetch the values, cannot assert those here.
            databaseAdapter.values(hash, keys, KeyFilterPredicate.ALLOW_ALL);
        }
    } finally {
        stopFlag.set(true);
        executor.shutdownNow();
        // 30 seconds is long, but necessary to let transactional databases detect deadlocks, which
        // cause Nessie-commit-retries.
        assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue();
        System.out.printf("AbstractConcurrency.concurrency - %s : Commits OK: %s  Retry-Failures: %s%n", variation, commitsOK, retryFailures);
        System.out.printf("AbstractConcurrency.concurrency - %s : try-loop success: count: %6d  retries: %6d  total-time-millis: %d%n", variation, (long) DatabaseAdapterMetrics.tryLoopCounts("success").count(), (long) DatabaseAdapterMetrics.tryLoopRetries("success").count(), (long) DatabaseAdapterMetrics.tryLoopDuration("success").totalTime(TimeUnit.MILLISECONDS));
        System.out.printf("AbstractConcurrency.concurrency - %s : try-loop failure: count: %6d  retries: %6d  total-time-millis: %d%n", variation, (long) DatabaseAdapterMetrics.tryLoopCounts("fail").count(), (long) DatabaseAdapterMetrics.tryLoopRetries("fail").count(), (long) DatabaseAdapterMetrics.tryLoopDuration("fail").totalTime(TimeUnit.MILLISECONDS));
        new ArrayList<>(globalRegistry.getRegistries()).forEach(globalRegistry::remove);
    }
}

Example 5 with BranchName

use of org.projectnessie.versioned.BranchName in project nessie by projectnessie.

the class AbstractDatabaseAdapterTest method assign.

@Test
void assign() throws Exception {
    BranchName main = BranchName.of("main");
    TagName tag = TagName.of("tag");
    TagName branch = TagName.of("branch");
    databaseAdapter.create(branch, databaseAdapter.hashOnReference(main, Optional.empty()));
    databaseAdapter.create(tag, databaseAdapter.hashOnReference(main, Optional.empty()));
    Hash beginning = databaseAdapter.hashOnReference(main, Optional.empty());
    Hash[] commits = new Hash[3];
    for (int i = 0; i < commits.length; i++) {
        commits[i] = databaseAdapter.commit(ImmutableCommitAttempt.builder().commitToBranch(main).commitMetaSerialized(ByteString.copyFromUtf8("commit meta " + i)).addPuts(KeyWithBytes.of(Key.of("bar", Integer.toString(i)), ContentId.of("contentId-" + i), (byte) 0, ByteString.copyFromUtf8("hello " + i))).build());
    }
    Hash expect = beginning;
    for (Hash commit : commits) {
        assertThat(Arrays.asList(databaseAdapter.hashOnReference(branch, Optional.empty()), databaseAdapter.hashOnReference(tag, Optional.empty()))).containsExactly(expect, expect);
        databaseAdapter.assign(tag, Optional.of(expect), commit);
        databaseAdapter.assign(branch, Optional.of(expect), commit);
        expect = commit;
    }
    assertThat(Arrays.asList(databaseAdapter.hashOnReference(branch, Optional.empty()), databaseAdapter.hashOnReference(tag, Optional.empty()))).containsExactly(commits[commits.length - 1], commits[commits.length - 1]);
}