use of org.projectnessie.versioned.testworker.WithGlobalStateContent in project nessie by projectnessie.
the class AbstractCompactGlobalLog method commitForGlobalLogCompaction.
private void commitForGlobalLogCompaction(int commits, ContentIdEmitter contentIdEmitter, BranchName branch, Map<ContentId, ByteString> currentGlobal, int i) throws ReferenceConflictException, ReferenceNotFoundException {
ContentId contentId = contentIdEmitter.contentIdForCommit();
Key key = Key.of("commit", Integer.toString(i));
WithGlobalStateContent c = WithGlobalStateContent.withGlobal("state for #" + i + " of " + commits, "value for #" + i + " of " + commits, contentId.getId());
byte payload = SimpleStoreWorker.INSTANCE.getPayload(c);
ByteString global = SimpleStoreWorker.INSTANCE.toStoreGlobalState(c);
ImmutableCommitAttempt.Builder commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("commit#" + i)).addPuts(KeyWithBytes.of(key, contentId, payload, SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c))).putGlobal(contentId, global);
currentGlobal.put(contentId, global);
databaseAdapter.commit(commit.build());
}
use of org.projectnessie.versioned.testworker.WithGlobalStateContent in project nessie by projectnessie.
the class AbstractConcurrency method concurrency.
@ParameterizedTest
@MethodSource("concurrencyVariations")
void concurrency(Variation variation) throws Exception {
new ArrayList<>(globalRegistry.getRegistries()).forEach(globalRegistry::remove);
globalRegistry.add(new SimpleMeterRegistry());
ExecutorService executor = Executors.newFixedThreadPool(variation.threads);
AtomicInteger commitsOK = new AtomicInteger();
AtomicInteger retryFailures = new AtomicInteger();
AtomicBoolean stopFlag = new AtomicBoolean();
List<Runnable> tasks = new ArrayList<>(variation.threads);
Map<Key, ContentId> keyToContentId = new HashMap<>();
Map<ContentId, ByteString> globalStates = new ConcurrentHashMap<>();
Map<BranchName, Map<Key, ByteString>> onRefStates = new ConcurrentHashMap<>();
try {
CountDownLatch startLatch = new CountDownLatch(1);
Map<BranchName, Set<Key>> keysPerBranch = new HashMap<>();
for (int i = 0; i < variation.threads; i++) {
BranchName branch = BranchName.of("concurrency-" + ((variation.singleBranch ? "shared" : i)));
List<Key> keys = new ArrayList<>(variation.tables);
for (int k = 0; k < variation.tables; k++) {
String variationKey = variation.sharedKeys ? "shared" : Integer.toString(i);
Key key = Key.of("some", "key", variationKey, "table-" + k);
keys.add(key);
keyToContentId.put(key, ContentId.of(String.format("%s-table-%d", variationKey, k)));
keysPerBranch.computeIfAbsent(branch, x -> new HashSet<>()).add(key);
}
tasks.add(() -> {
try {
assertThat(startLatch.await(2, TimeUnit.SECONDS)).isTrue();
for (int commit = 0; ; commit++) {
if (stopFlag.get()) {
break;
}
List<ByteString> currentStates = databaseAdapter.values(databaseAdapter.hashOnReference(branch, Optional.empty()), keys, KeyFilterPredicate.ALLOW_ALL).values().stream().map(ContentAndState::getGlobalState).collect(Collectors.toList());
ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder();
for (int ki = 0; ki < keys.size(); ki++) {
Key key = keys.get(ki);
ContentId contentId = keyToContentId.get(key);
String newGlobal = Integer.toString(Integer.parseInt(currentStates.get(ki).toStringUtf8()) + 1);
WithGlobalStateContent c = WithGlobalStateContent.withGlobal(newGlobal, "", contentId.getId());
commitAttempt.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(c));
if (!variation.sharedKeys) {
commitAttempt.putExpectedStates(contentId, Optional.of(currentStates.get(ki)));
}
commitAttempt.addPuts(KeyWithBytes.of(keys.get(ki), contentId, SimpleStoreWorker.INSTANCE.getPayload(c), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c)));
}
try {
commitAttempt.commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("commit #" + commit + " to " + branch.getName() + " something " + ThreadLocalRandom.current().nextLong()));
commitAndRecord(globalStates, onRefStates, branch, commitAttempt);
commitsOK.incrementAndGet();
} catch (ReferenceRetryFailureException retry) {
retryFailures.incrementAndGet();
}
}
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
});
}
for (Entry<BranchName, Set<Key>> branchKeys : keysPerBranch.entrySet()) {
BranchName branch = branchKeys.getKey();
databaseAdapter.create(branch, databaseAdapter.hashOnReference(BranchName.of("main"), Optional.empty()));
ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder().commitToBranch(branchKeys.getKey()).commitMetaSerialized(ByteString.copyFromUtf8("initial commit for " + branch.getName()));
for (Key k : branchKeys.getValue()) {
ContentId contentId = keyToContentId.get(k);
WithGlobalStateContent c = WithGlobalStateContent.withGlobal("0", "", contentId.getId());
commitAttempt.addPuts(KeyWithBytes.of(k, contentId, SimpleStoreWorker.INSTANCE.getPayload(c), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c)));
commitAttempt.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(c));
}
commitAndRecord(globalStates, onRefStates, branch, commitAttempt);
}
// Submit all 'Runnable's as 'CompletableFuture's and construct a combined 'CompletableFuture'
// that we can wait for.
CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(tasks.stream().map(r -> CompletableFuture.runAsync(r, executor)).toArray((IntFunction<CompletableFuture<?>[]>) CompletableFuture[]::new));
startLatch.countDown();
Thread.sleep(2_000);
stopFlag.set(true);
// 30 seconds is long, but necessary to let transactional databases detect deadlocks, which
// cause Nessie-commit-retries.
combinedFuture.get(30, TimeUnit.SECONDS);
for (Entry<BranchName, Set<Key>> branchKeys : keysPerBranch.entrySet()) {
BranchName branch = branchKeys.getKey();
Hash hash = databaseAdapter.hashOnReference(branch, Optional.empty());
ArrayList<Key> keys = new ArrayList<>(branchKeys.getValue());
// Note: only fetch the values, cannot assert those here.
databaseAdapter.values(hash, keys, KeyFilterPredicate.ALLOW_ALL);
}
} finally {
stopFlag.set(true);
executor.shutdownNow();
// 30 seconds is long, but necessary to let transactional databases detect deadlocks, which
// cause Nessie-commit-retries.
assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue();
System.out.printf("AbstractConcurrency.concurrency - %s : Commits OK: %s Retry-Failures: %s%n", variation, commitsOK, retryFailures);
System.out.printf("AbstractConcurrency.concurrency - %s : try-loop success: count: %6d retries: %6d total-time-millis: %d%n", variation, (long) DatabaseAdapterMetrics.tryLoopCounts("success").count(), (long) DatabaseAdapterMetrics.tryLoopRetries("success").count(), (long) DatabaseAdapterMetrics.tryLoopDuration("success").totalTime(TimeUnit.MILLISECONDS));
System.out.printf("AbstractConcurrency.concurrency - %s : try-loop failure: count: %6d retries: %6d total-time-millis: %d%n", variation, (long) DatabaseAdapterMetrics.tryLoopCounts("fail").count(), (long) DatabaseAdapterMetrics.tryLoopRetries("fail").count(), (long) DatabaseAdapterMetrics.tryLoopDuration("fail").totalTime(TimeUnit.MILLISECONDS));
new ArrayList<>(globalRegistry.getRegistries()).forEach(globalRegistry::remove);
}
}
use of org.projectnessie.versioned.testworker.WithGlobalStateContent in project nessie by projectnessie.
the class AbstractManyCommits method verify.
private void verify(int i, int numCommits, BranchName branch, Hash commit, ContentId contentId) {
Key key = Key.of("many", "commits", Integer.toString(numCommits));
try {
commit = databaseAdapter.hashOnReference(branch, Optional.of(commit));
} catch (ReferenceNotFoundException e) {
throw new RuntimeException(e);
}
try {
Map<Key, ContentAndState<ByteString>> values = databaseAdapter.values(commit, Collections.singletonList(key), KeyFilterPredicate.ALLOW_ALL);
WithGlobalStateContent expected = WithGlobalStateContent.withGlobal("state for #" + (numCommits - 1) + " of " + numCommits, "value for #" + i + " of " + numCommits, contentId.getId());
ByteString expectValue = SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(expected);
ByteString expectState = SimpleStoreWorker.INSTANCE.toStoreGlobalState(expected);
ContentAndState<ByteString> expect = ContentAndState.of(expectValue, expectState);
assertThat(values).containsExactly(Maps.immutableEntry(key, expect));
} catch (ReferenceNotFoundException e) {
throw new RuntimeException(e);
}
try (Stream<KeyListEntry> keys = databaseAdapter.keys(commit, KeyFilterPredicate.ALLOW_ALL)) {
assertThat(keys.map(KeyListEntry::getKey)).containsExactly(key);
} catch (ReferenceNotFoundException e) {
throw new RuntimeException(e);
}
}
use of org.projectnessie.versioned.testworker.WithGlobalStateContent in project nessie by projectnessie.
the class AbstractManyCommits method manyCommits.
@ParameterizedTest
@ValueSource(ints = { 0, 1, 19, 20, 21, 39, 40, 41, 49, 50, 51, 255, 256, 257, 500 })
// Note: 1000 commits is quite the max that in-JVM H2 database can handle
void manyCommits(int numCommits) throws Exception {
BranchName branch = BranchName.of("manyCommits-" + numCommits);
databaseAdapter.create(branch, databaseAdapter.hashOnReference(BranchName.of("main"), Optional.empty()));
Hash[] commits = new Hash[numCommits];
ContentId fixed = ContentId.of("FIXED");
for (int i = 0; i < numCommits; i++) {
Key key = Key.of("many", "commits", Integer.toString(numCommits));
WithGlobalStateContent c = WithGlobalStateContent.withGlobal("state for #" + i + " of " + numCommits, "value for #" + i + " of " + numCommits, fixed.getId());
byte payload = SimpleStoreWorker.INSTANCE.getPayload(c);
ImmutableCommitAttempt.Builder commit = ImmutableCommitAttempt.builder().commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("commit #" + i + " of " + numCommits)).addPuts(KeyWithBytes.of(key, fixed, payload, SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c))).putGlobal(fixed, SimpleStoreWorker.INSTANCE.toStoreGlobalState(c));
if (i > 0) {
WithGlobalStateContent expected = WithGlobalStateContent.withGlobal("state for #" + (i - 1) + " of " + numCommits, "value for #" + (i - 1) + " of " + numCommits, fixed.getId());
commit.putExpectedStates(fixed, Optional.of(SimpleStoreWorker.INSTANCE.toStoreGlobalState(expected)));
}
Hash hash = databaseAdapter.commit(commit.build());
commits[i] = hash;
try (Stream<ContentIdAndBytes> globals = databaseAdapter.globalContent(Collections.singleton(fixed))) {
WithGlobalStateContent expected = WithGlobalStateContent.withGlobal("state for #" + i + " of " + numCommits, "value for #" + i + " of " + numCommits, fixed.getId());
assertThat(globals).containsExactly(ContentIdAndBytes.of(fixed, SimpleStoreWorker.INSTANCE.toStoreGlobalState(expected)));
}
}
try (Stream<CommitLogEntry> log = databaseAdapter.commitLog(databaseAdapter.hashOnReference(branch, Optional.empty()))) {
assertThat(log.count()).isEqualTo(numCommits);
}
ExecutorService executor = Executors.newFixedThreadPool(Math.max(4, Runtime.getRuntime().availableProcessors()));
try {
CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(IntStream.range(0, numCommits).mapToObj(i -> (Runnable) () -> verify(i, numCommits, branch, commits[i], fixed)).map(r -> CompletableFuture.runAsync(r, executor)).toArray((IntFunction<CompletableFuture<?>[]>) CompletableFuture[]::new));
combinedFuture.get();
} finally {
executor.shutdown();
}
databaseAdapter.delete(branch, Optional.empty());
}
use of org.projectnessie.versioned.testworker.WithGlobalStateContent in project nessie by projectnessie.
the class CommitBench method doCommit.
private void doCommit(BenchmarkParam bp, BranchName branch, List<Key> keys, Map<Key, String> contentIds) throws Exception {
Map<Key, BaseContent> contentByKey = bp.versionStore.getValues(branch, keys);
try {
List<Operation<BaseContent>> operations = new ArrayList<>(bp.tablesPerCommit);
for (int i = 0; i < bp.tablesPerCommit; i++) {
Key key = keys.get(i);
BaseContent value = contentByKey.get(key);
if (value == null) {
throw new RuntimeException("no value for key " + key + " in " + branch);
}
String currentState = ((WithGlobalStateContent) value).getGlobal();
String newGlobalState = Integer.toString(Integer.parseInt(currentState) + 1);
String contentId = contentIds.get(key);
operations.add(Put.of(key, // hashes, because parent, "content", key are all the same.
withGlobal(newGlobalState, "commit value " + ThreadLocalRandom.current().nextLong(), contentId), withGlobal(currentState, "foo", contentId)));
}
bp.versionStore.commit(branch, Optional.empty(), commitMessage("commit meta data"), operations);
bp.success.incrementAndGet();
} catch (ReferenceRetryFailureException e) {
bp.retryFailures.incrementAndGet();
} catch (ReferenceConflictException e) {
bp.conflictsFailures.incrementAndGet();
}
}
Aggregations