Examples with GlobalLogCompactionParams org.projectnessie.versioned.persist.adapter.GlobalLogCompactionParams used on opensource projects

Example 1 with GlobalLogCompactionParams

use of org.projectnessie.versioned.persist.adapter.GlobalLogCompactionParams in project nessie by projectnessie.

the class AbstractCompactGlobalLog method compactGlobalLog.

@ParameterizedTest
@MethodSource("compactGlobalLog")
public void compactGlobalLog(int commits, int numContentIds) throws Exception {
    RepoMaintenanceParams repoMaintenanceParams = RepoMaintenanceParams.builder().globalLogCompactionParams(GlobalLogCompactionParams.builder().noCompactionWhenCompactedWithin(20).noCompactionUpToLength(20).build()).build();
    Map<String, Map<String, String>> statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    ContentIdEmitter contentIdEmitter = new ContentIdEmitter(numContentIds);
    // If there's no statistics entry for global-log-compaction, then it's not a non-transactional
    // database adapter, so no global-log to compact.
    assumeThat(statistics).containsKey("compactGlobalLog");
    // An "empty repository" should not require compaction
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    BranchName branch = BranchName.of("compactGlobalLog");
    Map<ContentId, ByteString> currentGlobal = new HashMap<>();
    databaseAdapter.create(branch, databaseAdapter.noAncestorHash());
    Runnable verify = () -> {
        try (Stream<ContentIdAndBytes> globals = databaseAdapter.globalContent(contentIdEmitter.allAsSet())) {
            assertThat(globals).hasSize(contentIdEmitter.size()).allSatisfy(cb -> assertThat(currentGlobal.get(cb.getContentId())).isEqualTo(cb.getValue()));
        }
    };
    for (int i = 0; i < commits; i++) {
        commitForGlobalLogCompaction(commits, contentIdEmitter, branch, currentGlobal, i);
    }
    // Verify
    verify.run();
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "true").containsEntry("entries.puts", Long.toString(commits)).containsEntry("entries.uniquePuts", Long.toString(contentIdEmitter.size())).containsEntry("entries.read", Long.toString(commits + 2)).containsEntry("entries.read.total", Long.toString(commits + 2));
    // Verify again
    verify.run();
    // Compact again, compaction must not run, because there is at least one compacted
    // global-log-entry in the first page (above only added 5 "uncompacted" global-log-entries).
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    // Add some more commits, but not enough to trigger compaction
    int additionalCommits = 5;
    for (int i = 0; i < additionalCommits; i++) {
        commitForGlobalLogCompaction(commits + additionalCommits, contentIdEmitter, branch, currentGlobal, i + commits);
    }
    // Compact again, compaction must not run, because there is at least one compacted
    // global-log-entry in the first page (above only added 5 "uncompacted" global-log-entries).
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    // Add some more commits, enough to trigger compaction again
    int additionalCommits2 = 15;
    for (int i = 0; i < additionalCommits2; i++) {
        commitForGlobalLogCompaction(commits + additionalCommits + additionalCommits2, contentIdEmitter, branch, currentGlobal, i + commits + additionalCommits);
    }
    // Compact again, compaction must run, because there is no compacted global-log-entry in the
    // first page of the global log.
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "true").containsEntry("entries.uniquePuts", Long.toString(contentIdEmitter.size()));
    // Verify again
    verify.run();
}

Example 2 with GlobalLogCompactionParams

use of org.projectnessie.versioned.persist.adapter.GlobalLogCompactionParams in project nessie by projectnessie.

the class RepoMaintenance method call.

@Override
public Integer call() {
    warnOnInMemory();
    ImmutableGlobalLogCompactionParams.Builder globalLogCompactionParams = GlobalLogCompactionParams.builder();
    if (noCompactionUpToLength != null) {
        globalLogCompactionParams.noCompactionUpToLength(noCompactionUpToLength);
    }
    if (noCompactionWhenCompactedWithin != null) {
        globalLogCompactionParams.noCompactionWhenCompactedWithin(noCompactionWhenCompactedWithin);
    }
    PrintWriter out = spec.commandLine().getOut();
    out.println("Running repository maintenance...");
    long t0 = System.nanoTime();
    Map<String, Map<String, String>> statistics = databaseAdapter.repoMaintenance(RepoMaintenanceParams.builder().globalLogCompactionParams(globalLogCompactionParams.build()).build());
    Duration duration = Duration.ofNanos(System.nanoTime() - t0);
    out.printf("Finished after %s%n", duration);
    statistics.entrySet().stream().sorted(comparingByKey()).forEach(top -> {
        out.printf("%nCategory '%s':%n", top.getKey());
        top.getValue().entrySet().stream().sorted(comparingByKey()).map(e -> String.format("    %-30s: %s", e.getKey(), e.getValue())).forEach(out::println);
    });
    return 0;
}