Examples with ContentIdAndBytes org.projectnessie.versioned.persist.adapter.ContentIdAndBytes used on opensource projects

Example 1 with ContentIdAndBytes

use of org.projectnessie.versioned.persist.adapter.ContentIdAndBytes in project nessie by projectnessie.

the class AbstractCompactGlobalLog method compactGlobalLog.

@ParameterizedTest
@MethodSource("compactGlobalLog")
public void compactGlobalLog(int commits, int numContentIds) throws Exception {
    RepoMaintenanceParams repoMaintenanceParams = RepoMaintenanceParams.builder().globalLogCompactionParams(GlobalLogCompactionParams.builder().noCompactionWhenCompactedWithin(20).noCompactionUpToLength(20).build()).build();
    Map<String, Map<String, String>> statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    ContentIdEmitter contentIdEmitter = new ContentIdEmitter(numContentIds);
    // If there's no statistics entry for global-log-compaction, then it's not a non-transactional
    // database adapter, so no global-log to compact.
    assumeThat(statistics).containsKey("compactGlobalLog");
    // An "empty repository" should not require compaction
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    BranchName branch = BranchName.of("compactGlobalLog");
    Map<ContentId, ByteString> currentGlobal = new HashMap<>();
    databaseAdapter.create(branch, databaseAdapter.noAncestorHash());
    Runnable verify = () -> {
        try (Stream<ContentIdAndBytes> globals = databaseAdapter.globalContent(contentIdEmitter.allAsSet())) {
            assertThat(globals).hasSize(contentIdEmitter.size()).allSatisfy(cb -> assertThat(currentGlobal.get(cb.getContentId())).isEqualTo(cb.getValue()));
        }
    };
    for (int i = 0; i < commits; i++) {
        commitForGlobalLogCompaction(commits, contentIdEmitter, branch, currentGlobal, i);
    }
    // Verify
    verify.run();
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "true").containsEntry("entries.puts", Long.toString(commits)).containsEntry("entries.uniquePuts", Long.toString(contentIdEmitter.size())).containsEntry("entries.read", Long.toString(commits + 2)).containsEntry("entries.read.total", Long.toString(commits + 2));
    // Verify again
    verify.run();
    // Compact again, compaction must not run, because there is at least one compacted
    // global-log-entry in the first page (above only added 5 "uncompacted" global-log-entries).
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    // Add some more commits, but not enough to trigger compaction
    int additionalCommits = 5;
    for (int i = 0; i < additionalCommits; i++) {
        commitForGlobalLogCompaction(commits + additionalCommits, contentIdEmitter, branch, currentGlobal, i + commits);
    }
    // Compact again, compaction must not run, because there is at least one compacted
    // global-log-entry in the first page (above only added 5 "uncompacted" global-log-entries).
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    // Add some more commits, enough to trigger compaction again
    int additionalCommits2 = 15;
    for (int i = 0; i < additionalCommits2; i++) {
        commitForGlobalLogCompaction(commits + additionalCommits + additionalCommits2, contentIdEmitter, branch, currentGlobal, i + commits + additionalCommits);
    }
    // Compact again, compaction must run, because there is no compacted global-log-entry in the
    // first page of the global log.
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "true").containsEntry("entries.uniquePuts", Long.toString(contentIdEmitter.size()));
    // Verify again
    verify.run();
}

Example 2 with ContentIdAndBytes

use of org.projectnessie.versioned.persist.adapter.ContentIdAndBytes in project nessie by projectnessie.

the class AbstractManyCommits method manyCommits.

@ParameterizedTest
@ValueSource(ints = { 0, 1, 19, 20, 21, 39, 40, 41, 49, 50, 51, 255, 256, 257, 500 })
// Note: 1000 commits is quite the max that in-JVM H2 database can handle
void manyCommits(int numCommits) throws Exception {
    BranchName branch = BranchName.of("manyCommits-" + numCommits);
    databaseAdapter.create(branch, databaseAdapter.hashOnReference(BranchName.of("main"), Optional.empty()));
    Hash[] commits = new Hash[numCommits];
    ContentId fixed = ContentId.of("FIXED");
    for (int i = 0; i < numCommits; i++) {
        Key key = Key.of("many", "commits", Integer.toString(numCommits));
        WithGlobalStateContent c = WithGlobalStateContent.withGlobal("state for #" + i + " of " + numCommits, "value for #" + i + " of " + numCommits, fixed.getId());
        byte payload = SimpleStoreWorker.INSTANCE.getPayload(c);
        ImmutableCommitAttempt.Builder commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("commit #" + i + " of " + numCommits)).addPuts(KeyWithBytes.of(key, fixed, payload, SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c))).putGlobal(fixed, SimpleStoreWorker.INSTANCE.toStoreGlobalState(c));
        if (i > 0) {
            WithGlobalStateContent expected = WithGlobalStateContent.withGlobal("state for #" + (i - 1) + " of " + numCommits, "value for #" + (i - 1) + " of " + numCommits, fixed.getId());
            commit.putExpectedStates(fixed, Optional.of(SimpleStoreWorker.INSTANCE.toStoreGlobalState(expected)));
        }
        Hash hash = databaseAdapter.commit(commit.build());
        commits[i] = hash;
        try (Stream<ContentIdAndBytes> globals = databaseAdapter.globalContent(Collections.singleton(fixed))) {
            WithGlobalStateContent expected = WithGlobalStateContent.withGlobal("state for #" + i + " of " + numCommits, "value for #" + i + " of " + numCommits, fixed.getId());
            assertThat(globals).containsExactly(ContentIdAndBytes.of(fixed, SimpleStoreWorker.INSTANCE.toStoreGlobalState(expected)));
        }
    }
    try (Stream<CommitLogEntry> log = databaseAdapter.commitLog(databaseAdapter.hashOnReference(branch, Optional.empty()))) {
        assertThat(log.count()).isEqualTo(numCommits);
    }
    ExecutorService executor = Executors.newFixedThreadPool(Math.max(4, Runtime.getRuntime().availableProcessors()));
    try {
        CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(IntStream.range(0, numCommits).mapToObj(i -> (Runnable) () -> verify(i, numCommits, branch, commits[i], fixed)).map(r -> CompletableFuture.runAsync(r, executor)).toArray((IntFunction<CompletableFuture<?>[]>) CompletableFuture[]::new));
        combinedFuture.get();
    } finally {
        executor.shutdown();
    }
    databaseAdapter.delete(branch, Optional.empty());
}

Example 3 with ContentIdAndBytes

use of org.projectnessie.versioned.persist.adapter.ContentIdAndBytes in project nessie by projectnessie.

the class AbstractGlobalStates method globalStates.

/**
 * Rudimentary test for Nessie-GC related basic operations to collect all globally known keys and
 * the global-state-logs.
 */
@ParameterizedTest
@MethodSource("globalStatesParams")
void globalStates(GlobalStateParam param) throws Exception {
    List<BranchName> branches = IntStream.range(0, param.branches).mapToObj(i -> BranchName.of("globalStates-" + i)).collect(Collectors.toList());
    Map<BranchName, Hash> heads = branches.stream().collect(Collectors.toMap(b -> b, b -> catchingFunction(() -> databaseAdapter.create(b, databaseAdapter.hashOnReference(BranchName.of("main"), Optional.empty())))));
    Map<ContentId, ByteString> currentStates = new HashMap<>();
    Set<Key> keys = IntStream.range(0, param.tables).mapToObj(i -> Key.of("table", Integer.toString(i))).collect(Collectors.toSet());
    Set<ContentId> usedContentIds = new HashSet<>();
    Map<ContentId, ByteString> expectedGlobalStates = new HashMap<>();
    for (int commit = 0; commit < param.commitsPerBranch; commit++) {
        for (BranchName branch : branches) {
            ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder().commitToBranch(branch).expectedHead(Optional.of(heads.get(branch))).commitMetaSerialized(ByteString.copyFromUtf8("some commit#" + commit + " branch " + branch.getName()));
            for (Key key : keys) {
                if (param.tableCommitProbability == 1.0f || ThreadLocalRandom.current().nextDouble(0d, 1d) <= param.tableCommitProbability) {
                    String state = "state-commit-" + commit + "+" + key;
                    String value = "value-commit-" + commit + "+" + key;
                    ContentId contentId = ContentId.of(key.toString() + "-" + branch.getName());
                    ByteString put = ByteString.copyFromUtf8(value);
                    ByteString global = ByteString.copyFromUtf8(state);
                    commitAttempt.putExpectedStates(contentId, Optional.ofNullable(currentStates.get(contentId))).putGlobal(contentId, global).addPuts(KeyWithBytes.of(key, contentId, (byte) 0, put));
                    expectedGlobalStates.put(contentId, global);
                    usedContentIds.add(contentId);
                    currentStates.put(contentId, global);
                }
            }
            ImmutableCommitAttempt attempt = commitAttempt.build();
            if (!attempt.getPuts().isEmpty()) {
                heads.put(branch, databaseAdapter.commit(attempt));
            }
        }
    }
    // verify that all global-state keys (== Key + content-id) are returned (in any order)
    try (Stream<ContentId> globalKeys = databaseAdapter.globalKeys()) {
        assertThat(globalKeys).containsExactlyInAnyOrderElementsOf(expectedGlobalStates.keySet());
    }
    try (Stream<ContentIdAndBytes> allStates = databaseAdapter.globalContent(expectedGlobalStates.keySet())) {
        List<ContentIdAndBytes> all = allStates.collect(Collectors.toList());
        // verify that the global-state-log returns all keys (in any order)
        assertThat(all.stream().map(ContentIdAndBytes::getContentId).distinct()).containsExactlyInAnyOrderElementsOf(usedContentIds);
        // verify that the global-state-log returns all content-ids (in any order)
        assertThat(all.stream().map(ContentIdAndBytes::getContentId).distinct()).containsExactlyInAnyOrderElementsOf(currentStates.keySet());
        Collection<ByteString> allExpected = expectedGlobalStates.values();
        // verify that the global-state-log returns all state-values
        assertThat(all.stream().map(ContentIdAndBytes::getValue)).containsExactlyInAnyOrderElementsOf(allExpected);
    }
}