use of org.projectnessie.versioned.persist.adapter.ContentId in project nessie by projectnessie.
the class PersistVersionStore method commit.
@Override
public Hash commit(@Nonnull BranchName branch, @Nonnull Optional<Hash> expectedHead, @Nonnull METADATA metadata, @Nonnull List<Operation<CONTENT>> operations, @Nonnull Callable<Void> validator) throws ReferenceNotFoundException, ReferenceConflictException {
ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder().commitToBranch(branch).expectedHead(expectedHead).commitMetaSerialized(serializeMetadata(metadata)).validator(validator);
for (Operation<CONTENT> operation : operations) {
if (operation instanceof Put) {
Put<CONTENT> op = (Put<CONTENT>) operation;
ContentId contentId = ContentId.of(storeWorker.getId(op.getValue()));
commitAttempt.addPuts(KeyWithBytes.of(op.getKey(), contentId, storeWorker.getPayload(op.getValue()), storeWorker.toStoreOnReferenceState(op.getValue())));
if (storeWorker.requiresGlobalState(op.getValue())) {
ByteString newState = storeWorker.toStoreGlobalState(op.getValue());
Optional<ByteString> expectedValue;
if (op.getExpectedValue() != null) {
if (storeWorker.getType(op.getValue()) != storeWorker.getType(op.getExpectedValue())) {
throw new IllegalArgumentException(String.format("Content-type for conditional put-operation for key '%s' for 'value' and 'expectedValue' must be the same, but are '%s' and '%s'.", op.getKey(), storeWorker.getType(op.getValue()), storeWorker.getType(op.getExpectedValue())));
}
if (!contentId.equals(ContentId.of(storeWorker.getId(op.getExpectedValue())))) {
throw new IllegalArgumentException(String.format("Conditional put-operation key '%s' has different content-ids.", op.getKey()));
}
expectedValue = Optional.of(storeWorker.toStoreGlobalState(op.getExpectedValue()));
} else {
expectedValue = Optional.empty();
}
commitAttempt.putExpectedStates(contentId, expectedValue);
commitAttempt.putGlobal(contentId, newState);
} else {
if (op.getExpectedValue() != null) {
throw new IllegalArgumentException(String.format("Content-type '%s' for put-operation for key '%s' does not support global state, expected-value not supported for this content-type.", storeWorker.getType(op.getValue()), op.getKey()));
}
}
} else if (operation instanceof Delete) {
commitAttempt.addDeletes(operation.getKey());
} else if (operation instanceof Unchanged) {
commitAttempt.addUnchanged(operation.getKey());
} else {
throw new IllegalArgumentException(String.format("Unknown operation type '%s'", operation));
}
}
return databaseAdapter.commit(commitAttempt.build());
}
use of org.projectnessie.versioned.persist.adapter.ContentId in project nessie by projectnessie.
the class AbstractCommitScenarios method commitRenameTable.
/**
* Test whether a "table rename operation" works as expected.
*
* <p>A "table rename" is effectively a remove-operation plus a put-operation with a different key
* but the same content-id.
*
* <p>Parameterized to force operations to cross/not-cross the number of commits between "full key
* lists" and whether global-state shall be used.
*/
@ParameterizedTest
@MethodSource("commitRenameTableParams")
void commitRenameTable(RenameTable param) throws Exception {
BranchName branch = BranchName.of("main");
Key dummyKey = Key.of("dummy");
Key oldKey = Key.of("hello", "table");
Key newKey = Key.of("new", "name");
ContentId contentId = ContentId.of("id-42");
IntFunction<Hash> performDummyCommit = i -> {
try {
return databaseAdapter.commit(ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("dummy commit meta " + i)).addUnchanged(dummyKey).build());
} catch (ReferenceNotFoundException | ReferenceConflictException e) {
throw new RuntimeException(e);
}
};
List<Hash> beforeInitial = IntStream.range(0, param.setupCommits).mapToObj(performDummyCommit).collect(Collectors.toList());
ImmutableCommitAttempt.Builder commit;
BaseContent initialContent;
BaseContent renamContent;
if (param.globalState) {
initialContent = WithGlobalStateContent.newWithGlobal("0", "initial commit content");
renamContent = WithGlobalStateContent.withGlobal("0", "rename commit content", initialContent.getId());
} else {
initialContent = OnRefOnly.newOnRef("initial commit content");
renamContent = OnRefOnly.onRef("rename commit content", initialContent.getId());
}
byte payload = SimpleStoreWorker.INSTANCE.getPayload(initialContent);
commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("initial commit meta")).addPuts(KeyWithBytes.of(oldKey, contentId, payload, SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(initialContent)));
if (param.globalState) {
commit.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(initialContent)).putExpectedStates(contentId, Optional.empty());
}
Hash hashInitial = databaseAdapter.commit(commit.build());
List<Hash> beforeRename = IntStream.range(0, param.afterInitialCommits).mapToObj(performDummyCommit).collect(Collectors.toList());
commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("rename table")).addDeletes(oldKey).addPuts(KeyWithBytes.of(newKey, contentId, payload, SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(renamContent)));
if (param.globalState) {
commit.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(renamContent)).putExpectedStates(contentId, Optional.of(SimpleStoreWorker.INSTANCE.toStoreGlobalState(initialContent)));
}
Hash hashRename = databaseAdapter.commit(commit.build());
List<Hash> beforeDelete = IntStream.range(0, param.afterRenameCommits).mapToObj(performDummyCommit).collect(Collectors.toList());
commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("delete table")).addDeletes(newKey);
if (param.globalState) {
commit.putGlobal(contentId, ByteString.copyFromUtf8("0")).putExpectedStates(contentId, Optional.of(ByteString.copyFromUtf8("0")));
}
Hash hashDelete = databaseAdapter.commit(commit.build());
List<Hash> afterDelete = IntStream.range(0, param.afterDeleteCommits).mapToObj(performDummyCommit).collect(Collectors.toList());
int expectedCommitCount = 1;
// Verify that the commits before the initial put return _no_ keys
expectedCommitCount = renameCommitVerify(beforeInitial.stream(), expectedCommitCount, keys -> assertThat(keys).isEmpty());
// Verify that the commits since the initial put and before the rename-operation return the
// _old_ key
expectedCommitCount = renameCommitVerify(Stream.concat(Stream.of(hashInitial), beforeRename.stream()), expectedCommitCount, keys -> assertThat(keys).containsExactly(KeyListEntry.of(oldKey, contentId, payload, hashInitial)));
// Verify that the commits since the rename-operation and before the delete-operation return the
// _new_ key
expectedCommitCount = renameCommitVerify(Stream.concat(Stream.of(hashRename), beforeDelete.stream()), expectedCommitCount, keys -> assertThat(keys).containsExactly(KeyListEntry.of(newKey, contentId, payload, hashRename)));
// Verify that the commits since the delete-operation return _no_ keys
expectedCommitCount = renameCommitVerify(Stream.concat(Stream.of(hashDelete), afterDelete.stream()), expectedCommitCount, keys -> assertThat(keys).isEmpty());
assertThat(expectedCommitCount - 1).isEqualTo(param.setupCommits + 1 + param.afterInitialCommits + 1 + param.afterRenameCommits + 1 + param.afterDeleteCommits);
}
use of org.projectnessie.versioned.persist.adapter.ContentId in project nessie by projectnessie.
the class AbstractCompactGlobalLog method compactGlobalLog.
@ParameterizedTest
@MethodSource("compactGlobalLog")
public void compactGlobalLog(int commits, int numContentIds) throws Exception {
RepoMaintenanceParams repoMaintenanceParams = RepoMaintenanceParams.builder().globalLogCompactionParams(GlobalLogCompactionParams.builder().noCompactionWhenCompactedWithin(20).noCompactionUpToLength(20).build()).build();
Map<String, Map<String, String>> statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
ContentIdEmitter contentIdEmitter = new ContentIdEmitter(numContentIds);
// If there's no statistics entry for global-log-compaction, then it's not a non-transactional
// database adapter, so no global-log to compact.
assumeThat(statistics).containsKey("compactGlobalLog");
// An "empty repository" should not require compaction
assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
BranchName branch = BranchName.of("compactGlobalLog");
Map<ContentId, ByteString> currentGlobal = new HashMap<>();
databaseAdapter.create(branch, databaseAdapter.noAncestorHash());
Runnable verify = () -> {
try (Stream<ContentIdAndBytes> globals = databaseAdapter.globalContent(contentIdEmitter.allAsSet())) {
assertThat(globals).hasSize(contentIdEmitter.size()).allSatisfy(cb -> assertThat(currentGlobal.get(cb.getContentId())).isEqualTo(cb.getValue()));
}
};
for (int i = 0; i < commits; i++) {
commitForGlobalLogCompaction(commits, contentIdEmitter, branch, currentGlobal, i);
}
// Verify
verify.run();
statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "true").containsEntry("entries.puts", Long.toString(commits)).containsEntry("entries.uniquePuts", Long.toString(contentIdEmitter.size())).containsEntry("entries.read", Long.toString(commits + 2)).containsEntry("entries.read.total", Long.toString(commits + 2));
// Verify again
verify.run();
// Compact again, compaction must not run, because there is at least one compacted
// global-log-entry in the first page (above only added 5 "uncompacted" global-log-entries).
statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
// Add some more commits, but not enough to trigger compaction
int additionalCommits = 5;
for (int i = 0; i < additionalCommits; i++) {
commitForGlobalLogCompaction(commits + additionalCommits, contentIdEmitter, branch, currentGlobal, i + commits);
}
// Compact again, compaction must not run, because there is at least one compacted
// global-log-entry in the first page (above only added 5 "uncompacted" global-log-entries).
statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "false");
// Add some more commits, enough to trigger compaction again
int additionalCommits2 = 15;
for (int i = 0; i < additionalCommits2; i++) {
commitForGlobalLogCompaction(commits + additionalCommits + additionalCommits2, contentIdEmitter, branch, currentGlobal, i + commits + additionalCommits);
}
// Compact again, compaction must run, because there is no compacted global-log-entry in the
// first page of the global log.
statistics = databaseAdapter.repoMaintenance(repoMaintenanceParams);
assertThat(statistics).containsKey("compactGlobalLog").extracting("compactGlobalLog", InstanceOfAssertFactories.map(String.class, String.class)).containsEntry("compacted", "true").containsEntry("entries.uniquePuts", Long.toString(contentIdEmitter.size()));
// Verify again
verify.run();
}
use of org.projectnessie.versioned.persist.adapter.ContentId in project nessie by projectnessie.
the class AbstractCompactGlobalLog method commitForGlobalLogCompaction.
private void commitForGlobalLogCompaction(int commits, ContentIdEmitter contentIdEmitter, BranchName branch, Map<ContentId, ByteString> currentGlobal, int i) throws ReferenceConflictException, ReferenceNotFoundException {
ContentId contentId = contentIdEmitter.contentIdForCommit();
Key key = Key.of("commit", Integer.toString(i));
WithGlobalStateContent c = WithGlobalStateContent.withGlobal("state for #" + i + " of " + commits, "value for #" + i + " of " + commits, contentId.getId());
byte payload = SimpleStoreWorker.INSTANCE.getPayload(c);
ByteString global = SimpleStoreWorker.INSTANCE.toStoreGlobalState(c);
ImmutableCommitAttempt.Builder commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("commit#" + i)).addPuts(KeyWithBytes.of(key, contentId, payload, SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c))).putGlobal(contentId, global);
currentGlobal.put(contentId, global);
databaseAdapter.commit(commit.build());
}
use of org.projectnessie.versioned.persist.adapter.ContentId in project nessie by projectnessie.
the class AbstractConcurrency method concurrency.
@ParameterizedTest
@MethodSource("concurrencyVariations")
void concurrency(Variation variation) throws Exception {
new ArrayList<>(globalRegistry.getRegistries()).forEach(globalRegistry::remove);
globalRegistry.add(new SimpleMeterRegistry());
ExecutorService executor = Executors.newFixedThreadPool(variation.threads);
AtomicInteger commitsOK = new AtomicInteger();
AtomicInteger retryFailures = new AtomicInteger();
AtomicBoolean stopFlag = new AtomicBoolean();
List<Runnable> tasks = new ArrayList<>(variation.threads);
Map<Key, ContentId> keyToContentId = new HashMap<>();
Map<ContentId, ByteString> globalStates = new ConcurrentHashMap<>();
Map<BranchName, Map<Key, ByteString>> onRefStates = new ConcurrentHashMap<>();
try {
CountDownLatch startLatch = new CountDownLatch(1);
Map<BranchName, Set<Key>> keysPerBranch = new HashMap<>();
for (int i = 0; i < variation.threads; i++) {
BranchName branch = BranchName.of("concurrency-" + ((variation.singleBranch ? "shared" : i)));
List<Key> keys = new ArrayList<>(variation.tables);
for (int k = 0; k < variation.tables; k++) {
String variationKey = variation.sharedKeys ? "shared" : Integer.toString(i);
Key key = Key.of("some", "key", variationKey, "table-" + k);
keys.add(key);
keyToContentId.put(key, ContentId.of(String.format("%s-table-%d", variationKey, k)));
keysPerBranch.computeIfAbsent(branch, x -> new HashSet<>()).add(key);
}
tasks.add(() -> {
try {
assertThat(startLatch.await(2, TimeUnit.SECONDS)).isTrue();
for (int commit = 0; ; commit++) {
if (stopFlag.get()) {
break;
}
List<ByteString> currentStates = databaseAdapter.values(databaseAdapter.hashOnReference(branch, Optional.empty()), keys, KeyFilterPredicate.ALLOW_ALL).values().stream().map(ContentAndState::getGlobalState).collect(Collectors.toList());
ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder();
for (int ki = 0; ki < keys.size(); ki++) {
Key key = keys.get(ki);
ContentId contentId = keyToContentId.get(key);
String newGlobal = Integer.toString(Integer.parseInt(currentStates.get(ki).toStringUtf8()) + 1);
WithGlobalStateContent c = WithGlobalStateContent.withGlobal(newGlobal, "", contentId.getId());
commitAttempt.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(c));
if (!variation.sharedKeys) {
commitAttempt.putExpectedStates(contentId, Optional.of(currentStates.get(ki)));
}
commitAttempt.addPuts(KeyWithBytes.of(keys.get(ki), contentId, SimpleStoreWorker.INSTANCE.getPayload(c), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c)));
}
try {
commitAttempt.commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("commit #" + commit + " to " + branch.getName() + " something " + ThreadLocalRandom.current().nextLong()));
commitAndRecord(globalStates, onRefStates, branch, commitAttempt);
commitsOK.incrementAndGet();
} catch (ReferenceRetryFailureException retry) {
retryFailures.incrementAndGet();
}
}
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
});
}
for (Entry<BranchName, Set<Key>> branchKeys : keysPerBranch.entrySet()) {
BranchName branch = branchKeys.getKey();
databaseAdapter.create(branch, databaseAdapter.hashOnReference(BranchName.of("main"), Optional.empty()));
ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder().commitToBranch(branchKeys.getKey()).commitMetaSerialized(ByteString.copyFromUtf8("initial commit for " + branch.getName()));
for (Key k : branchKeys.getValue()) {
ContentId contentId = keyToContentId.get(k);
WithGlobalStateContent c = WithGlobalStateContent.withGlobal("0", "", contentId.getId());
commitAttempt.addPuts(KeyWithBytes.of(k, contentId, SimpleStoreWorker.INSTANCE.getPayload(c), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c)));
commitAttempt.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(c));
}
commitAndRecord(globalStates, onRefStates, branch, commitAttempt);
}
// Submit all 'Runnable's as 'CompletableFuture's and construct a combined 'CompletableFuture'
// that we can wait for.
CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(tasks.stream().map(r -> CompletableFuture.runAsync(r, executor)).toArray((IntFunction<CompletableFuture<?>[]>) CompletableFuture[]::new));
startLatch.countDown();
Thread.sleep(2_000);
stopFlag.set(true);
// 30 seconds is long, but necessary to let transactional databases detect deadlocks, which
// cause Nessie-commit-retries.
combinedFuture.get(30, TimeUnit.SECONDS);
for (Entry<BranchName, Set<Key>> branchKeys : keysPerBranch.entrySet()) {
BranchName branch = branchKeys.getKey();
Hash hash = databaseAdapter.hashOnReference(branch, Optional.empty());
ArrayList<Key> keys = new ArrayList<>(branchKeys.getValue());
// Note: only fetch the values, cannot assert those here.
databaseAdapter.values(hash, keys, KeyFilterPredicate.ALLOW_ALL);
}
} finally {
stopFlag.set(true);
executor.shutdownNow();
// 30 seconds is long, but necessary to let transactional databases detect deadlocks, which
// cause Nessie-commit-retries.
assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue();
System.out.printf("AbstractConcurrency.concurrency - %s : Commits OK: %s Retry-Failures: %s%n", variation, commitsOK, retryFailures);
System.out.printf("AbstractConcurrency.concurrency - %s : try-loop success: count: %6d retries: %6d total-time-millis: %d%n", variation, (long) DatabaseAdapterMetrics.tryLoopCounts("success").count(), (long) DatabaseAdapterMetrics.tryLoopRetries("success").count(), (long) DatabaseAdapterMetrics.tryLoopDuration("success").totalTime(TimeUnit.MILLISECONDS));
System.out.printf("AbstractConcurrency.concurrency - %s : try-loop failure: count: %6d retries: %6d total-time-millis: %d%n", variation, (long) DatabaseAdapterMetrics.tryLoopCounts("fail").count(), (long) DatabaseAdapterMetrics.tryLoopRetries("fail").count(), (long) DatabaseAdapterMetrics.tryLoopDuration("fail").totalTime(TimeUnit.MILLISECONDS));
new ArrayList<>(globalRegistry.getRegistries()).forEach(globalRegistry::remove);
}
}
Aggregations