Search in sources :

Example 1 with GlobalLogCompactionParams

use of org.projectnessie.versioned.persist.adapter.GlobalLogCompactionParams in project nessie by projectnessie.

the class AbstractCompactGlobalLog method compactGlobalLog.

@ParameterizedTest
@MethodSource("compactGlobalLog")
public void compactGlobalLog(int commits, int numContentIds) throws Exception {
    RepoMaintenanceParams repoMaintenanceParams = RepoMaintenanceParams.builder().globalLogCompactionParams(GlobalLogCompactionParams.builder().noCompactionWhenCompactedWithin(20).noCompactionUpToLength(20).build()).build();
    Map<String, Map<String, String>> statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    ContentIdEmitter contentIdEmitter = new ContentIdEmitter(numContentIds);
    // If there's no statistics entry for global-log-compaction, then it's not a non-transactional
    // database adapter, so no global-log to compact.
    assumeThat(statistics).containsKey("compactGlobalLog");
    // An "empty repository" should not require compaction
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    BranchName branch = BranchName.of("compactGlobalLog");
    Map<ContentId, ByteString> currentGlobal = new HashMap<>();
    databaseAdapter.create(branch, databaseAdapter.noAncestorHash());
    Runnable verify = () -> {
        try (Stream<ContentIdAndBytes> globals = databaseAdapter.globalContent(contentIdEmitter.allAsSet())) {
            assertThat(globals).hasSize(contentIdEmitter.size()).allSatisfy(cb -> assertThat(currentGlobal.get(cb.getContentId())).isEqualTo(cb.getValue()));
        }
    };
    for (int i = 0; i < commits; i++) {
        commitForGlobalLogCompaction(commits, contentIdEmitter, branch, currentGlobal, i);
    }
    // Verify
    verify.run();
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "true").containsEntry("entries.puts", Long.toString(commits)).containsEntry("entries.uniquePuts", Long.toString(contentIdEmitter.size())).containsEntry("entries.read", Long.toString(commits + 2)).containsEntry("entries.read.total", Long.toString(commits + 2));
    // Verify again
    verify.run();
    // Compact again, compaction must not run, because there is at least one compacted
    // global-log-entry in the first page (above only added 5 "uncompacted" global-log-entries).
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    // Add some more commits, but not enough to trigger compaction
    int additionalCommits = 5;
    for (int i = 0; i < additionalCommits; i++) {
        commitForGlobalLogCompaction(commits + additionalCommits, contentIdEmitter, branch, currentGlobal, i + commits);
    }
    // Compact again, compaction must not run, because there is at least one compacted
    // global-log-entry in the first page (above only added 5 "uncompacted" global-log-entries).
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
    // Add some more commits, enough to trigger compaction again
    int additionalCommits2 = 15;
    for (int i = 0; i < additionalCommits2; i++) {
        commitForGlobalLogCompaction(commits + additionalCommits + additionalCommits2, contentIdEmitter, branch, currentGlobal, i + commits + additionalCommits);
    }
    // Compact again, compaction must run, because there is no compacted global-log-entry in the
    // first page of the global log.
    statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
    assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "true").containsEntry("entries.uniquePuts", Long.toString(contentIdEmitter.size()));
    // Verify again
    verify.run();
}
Also used : IntStream(java.util.stream.IntStream) InstanceOfAssertFactories(org.assertj.core.api.InstanceOfAssertFactories) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) HashMap(java.util.HashMap) HashSet(java.util.HashSet) RepoMaintenanceParams(org.projectnessie.versioned.persist.adapter.RepoMaintenanceParams) Map(java.util.Map) DatabaseAdapter(org.projectnessie.versioned.persist.adapter.DatabaseAdapter) MethodSource(org.junit.jupiter.params.provider.MethodSource) ContentIdAndBytes(org.projectnessie.versioned.persist.adapter.ContentIdAndBytes) WithGlobalStateContent(org.projectnessie.versioned.testworker.WithGlobalStateContent) Assumptions.assumeThat(org.assertj.core.api.Assumptions.assumeThat) Set(java.util.Set) Arguments(org.junit.jupiter.params.provider.Arguments) Key(org.projectnessie.versioned.Key) Collectors(java.util.stream.Collectors) ReferenceNotFoundException(org.projectnessie.versioned.ReferenceNotFoundException) KeyWithBytes(org.projectnessie.versioned.persist.adapter.KeyWithBytes) ByteString(com.google.protobuf.ByteString) BranchName(org.projectnessie.versioned.BranchName) ReferenceConflictException(org.projectnessie.versioned.ReferenceConflictException) List(java.util.List) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Stream(java.util.stream.Stream) ImmutableCommitAttempt(org.projectnessie.versioned.persist.adapter.ImmutableCommitAttempt) GlobalLogCompactionParams(org.projectnessie.versioned.persist.adapter.GlobalLogCompactionParams) SimpleStoreWorker(org.projectnessie.versioned.testworker.SimpleStoreWorker) ContentId(org.projectnessie.versioned.persist.adapter.ContentId) HashMap(java.util.HashMap) ByteString(com.google.protobuf.ByteString) ByteString(com.google.protobuf.ByteString) ContentId(org.projectnessie.versioned.persist.adapter.ContentId) IntStream(java.util.stream.IntStream) Stream(java.util.stream.Stream) BranchName(org.projectnessie.versioned.BranchName) RepoMaintenanceParams(org.projectnessie.versioned.persist.adapter.RepoMaintenanceParams) HashMap(java.util.HashMap) Map(java.util.Map) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) MethodSource(org.junit.jupiter.params.provider.MethodSource)

Example 2 with GlobalLogCompactionParams

use of org.projectnessie.versioned.persist.adapter.GlobalLogCompactionParams in project nessie by projectnessie.

the class RepoMaintenance method call.

@Override
public Integer call() {
    warnOnInMemory();
    ImmutableGlobalLogCompactionParams.Builder globalLogCompactionParams = GlobalLogCompactionParams.builder();
    if (noCompactionUpToLength != null) {
        globalLogCompactionParams.noCompactionUpToLength(noCompactionUpToLength);
    }
    if (noCompactionWhenCompactedWithin != null) {
        globalLogCompactionParams.noCompactionWhenCompactedWithin(noCompactionWhenCompactedWithin);
    }
    PrintWriter out = spec.commandLine().getOut();
    out.println("Running repository maintenance...");
    long t0 = System.nanoTime();
    Map<String, Map<String, String>> statistics = databaseAdapter.repoMaintenance(RepoMaintenanceParams.builder().globalLogCompactionParams(globalLogCompactionParams.build()).build());
    Duration duration = Duration.ofNanos(System.nanoTime() - t0);
    out.printf("Finished after %s%n", duration);
    statistics.entrySet().stream().sorted(comparingByKey()).forEach(top -> {
        out.printf("%nCategory '%s':%n", top.getKey());
        top.getValue().entrySet().stream().sorted(comparingByKey()).map(e -> String.format("    %-30s: %s", e.getKey(), e.getValue())).forEach(out::println);
    });
    return 0;
}
Also used : PrintWriter(java.io.PrintWriter) RepoMaintenanceParams(org.projectnessie.versioned.persist.adapter.RepoMaintenanceParams) Option(picocli.CommandLine.Option) Entry.comparingByKey(java.util.Map.Entry.comparingByKey) ImmutableGlobalLogCompactionParams(org.projectnessie.versioned.persist.adapter.ImmutableGlobalLogCompactionParams) Duration(java.time.Duration) Map(java.util.Map) GlobalLogCompactionParams(org.projectnessie.versioned.persist.adapter.GlobalLogCompactionParams) Command(picocli.CommandLine.Command) Duration(java.time.Duration) ImmutableGlobalLogCompactionParams(org.projectnessie.versioned.persist.adapter.ImmutableGlobalLogCompactionParams) Map(java.util.Map) PrintWriter(java.io.PrintWriter)

Aggregations

Map (java.util.Map)2 GlobalLogCompactionParams (org.projectnessie.versioned.persist.adapter.GlobalLogCompactionParams)2 RepoMaintenanceParams (org.projectnessie.versioned.persist.adapter.RepoMaintenanceParams)2 ByteString (com.google.protobuf.ByteString)1 PrintWriter (java.io.PrintWriter)1 Duration (java.time.Duration)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Entry.comparingByKey (java.util.Map.Entry.comparingByKey)1 Set (java.util.Set)1 Collectors (java.util.stream.Collectors)1 IntStream (java.util.stream.IntStream)1 Stream (java.util.stream.Stream)1 Assertions.assertThat (org.assertj.core.api.Assertions.assertThat)1 Assumptions.assumeThat (org.assertj.core.api.Assumptions.assumeThat)1 InstanceOfAssertFactories (org.assertj.core.api.InstanceOfAssertFactories)1 ParameterizedTest (org.junit.jupiter.params.ParameterizedTest)1 Arguments (org.junit.jupiter.params.provider.Arguments)1 MethodSource (org.junit.jupiter.params.provider.MethodSource)1