Search in sources :

Example 1 with CommitAttempt

use of org.projectnessie.versioned.persist.adapter.CommitAttempt in project nessie by projectnessie.

the class AbstractConcurrency method concurrency.

@ParameterizedTest
@MethodSource("concurrencyVariations")
void concurrency(Variation variation) throws Exception {
    new ArrayList<>(globalRegistry.getRegistries()).forEach(globalRegistry::remove);
    globalRegistry.add(new SimpleMeterRegistry());
    ExecutorService executor = Executors.newFixedThreadPool(variation.threads);
    AtomicInteger commitsOK = new AtomicInteger();
    AtomicInteger retryFailures = new AtomicInteger();
    AtomicBoolean stopFlag = new AtomicBoolean();
    List<Runnable> tasks = new ArrayList<>(variation.threads);
    Map<Key, ContentId> keyToContentId = new HashMap<>();
    Map<ContentId, ByteString> globalStates = new ConcurrentHashMap<>();
    Map<BranchName, Map<Key, ByteString>> onRefStates = new ConcurrentHashMap<>();
    try {
        CountDownLatch startLatch = new CountDownLatch(1);
        Map<BranchName, Set<Key>> keysPerBranch = new HashMap<>();
        for (int i = 0; i < variation.threads; i++) {
            BranchName branch = BranchName.of("concurrency-" + ((variation.singleBranch ? "shared" : i)));
            List<Key> keys = new ArrayList<>(variation.tables);
            for (int k = 0; k < variation.tables; k++) {
                String variationKey = variation.sharedKeys ? "shared" : Integer.toString(i);
                Key key = Key.of("some", "key", variationKey, "table-" + k);
                keys.add(key);
                keyToContentId.put(key, ContentId.of(String.format("%s-table-%d", variationKey, k)));
                keysPerBranch.computeIfAbsent(branch, x -> new HashSet<>()).add(key);
            }
            tasks.add(() -> {
                try {
                    assertThat(startLatch.await(2, TimeUnit.SECONDS)).isTrue();
                    for (int commit = 0; ; commit++) {
                        if (stopFlag.get()) {
                            break;
                        }
                        List<ByteString> currentStates = databaseAdapter.values(databaseAdapter.hashOnReference(branch, Optional.empty()), keys, KeyFilterPredicate.ALLOW_ALL).values().stream().map(ContentAndState::getGlobalState).collect(Collectors.toList());
                        ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder();
                        for (int ki = 0; ki < keys.size(); ki++) {
                            Key key = keys.get(ki);
                            ContentId contentId = keyToContentId.get(key);
                            String newGlobal = Integer.toString(Integer.parseInt(currentStates.get(ki).toStringUtf8()) + 1);
                            WithGlobalStateContent c = WithGlobalStateContent.withGlobal(newGlobal, "", contentId.getId());
                            commitAttempt.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(c));
                            if (!variation.sharedKeys) {
                                commitAttempt.putExpectedStates(contentId, Optional.of(currentStates.get(ki)));
                            }
                            commitAttempt.addPuts(KeyWithBytes.of(keys.get(ki), contentId, SimpleStoreWorker.INSTANCE.getPayload(c), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c)));
                        }
                        try {
                            commitAttempt.commitToBranch(branch).commitMetaSerialized(ByteString.copyFromUtf8("commit #" + commit + " to " + branch.getName() + " something " + ThreadLocalRandom.current().nextLong()));
                            commitAndRecord(globalStates, onRefStates, branch, commitAttempt);
                            commitsOK.incrementAndGet();
                        } catch (ReferenceRetryFailureException retry) {
                            retryFailures.incrementAndGet();
                        }
                    }
                } catch (Exception e) {
                    e.printStackTrace();
                    throw new RuntimeException(e);
                }
            });
        }
        for (Entry<BranchName, Set<Key>> branchKeys : keysPerBranch.entrySet()) {
            BranchName branch = branchKeys.getKey();
            databaseAdapter.create(branch, databaseAdapter.hashOnReference(BranchName.of("main"), Optional.empty()));
            ImmutableCommitAttempt.Builder commitAttempt = ImmutableCommitAttempt.builder().commitToBranch(branchKeys.getKey()).commitMetaSerialized(ByteString.copyFromUtf8("initial commit for " + branch.getName()));
            for (Key k : branchKeys.getValue()) {
                ContentId contentId = keyToContentId.get(k);
                WithGlobalStateContent c = WithGlobalStateContent.withGlobal("0", "", contentId.getId());
                commitAttempt.addPuts(KeyWithBytes.of(k, contentId, SimpleStoreWorker.INSTANCE.getPayload(c), SimpleStoreWorker.INSTANCE.toStoreOnReferenceState(c)));
                commitAttempt.putGlobal(contentId, SimpleStoreWorker.INSTANCE.toStoreGlobalState(c));
            }
            commitAndRecord(globalStates, onRefStates, branch, commitAttempt);
        }
        // Submit all 'Runnable's as 'CompletableFuture's and construct a combined 'CompletableFuture'
        // that we can wait for.
        CompletableFuture<Void> combinedFuture = CompletableFuture.allOf(tasks.stream().map(r -> CompletableFuture.runAsync(r, executor)).toArray((IntFunction<CompletableFuture<?>[]>) CompletableFuture[]::new));
        startLatch.countDown();
        Thread.sleep(2_000);
        stopFlag.set(true);
        // 30 seconds is long, but necessary to let transactional databases detect deadlocks, which
        // cause Nessie-commit-retries.
        combinedFuture.get(30, TimeUnit.SECONDS);
        for (Entry<BranchName, Set<Key>> branchKeys : keysPerBranch.entrySet()) {
            BranchName branch = branchKeys.getKey();
            Hash hash = databaseAdapter.hashOnReference(branch, Optional.empty());
            ArrayList<Key> keys = new ArrayList<>(branchKeys.getValue());
            // Note: only fetch the values, cannot assert those here.
            databaseAdapter.values(hash, keys, KeyFilterPredicate.ALLOW_ALL);
        }
    } finally {
        stopFlag.set(true);
        executor.shutdownNow();
        // 30 seconds is long, but necessary to let transactional databases detect deadlocks, which
        // cause Nessie-commit-retries.
        assertThat(executor.awaitTermination(30, TimeUnit.SECONDS)).isTrue();
        System.out.printf("AbstractConcurrency.concurrency - %s : Commits OK: %s  Retry-Failures: %s%n", variation, commitsOK, retryFailures);
        System.out.printf("AbstractConcurrency.concurrency - %s : try-loop success: count: %6d  retries: %6d  total-time-millis: %d%n", variation, (long) DatabaseAdapterMetrics.tryLoopCounts("success").count(), (long) DatabaseAdapterMetrics.tryLoopRetries("success").count(), (long) DatabaseAdapterMetrics.tryLoopDuration("success").totalTime(TimeUnit.MILLISECONDS));
        System.out.printf("AbstractConcurrency.concurrency - %s : try-loop failure: count: %6d  retries: %6d  total-time-millis: %d%n", variation, (long) DatabaseAdapterMetrics.tryLoopCounts("fail").count(), (long) DatabaseAdapterMetrics.tryLoopRetries("fail").count(), (long) DatabaseAdapterMetrics.tryLoopDuration("fail").totalTime(TimeUnit.MILLISECONDS));
        new ArrayList<>(globalRegistry.getRegistries()).forEach(globalRegistry::remove);
    }
}
Also used : SimpleMeterRegistry(io.micrometer.core.instrument.simple.SimpleMeterRegistry) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) ArrayList(java.util.ArrayList) HashSet(java.util.HashSet) ReferenceRetryFailureException(org.projectnessie.versioned.ReferenceRetryFailureException) KeyFilterPredicate(org.projectnessie.versioned.persist.adapter.KeyFilterPredicate) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) CommitAttempt(org.projectnessie.versioned.persist.adapter.CommitAttempt) Map(java.util.Map) ThreadLocalRandom(java.util.concurrent.ThreadLocalRandom) ExecutorService(java.util.concurrent.ExecutorService) DatabaseAdapter(org.projectnessie.versioned.persist.adapter.DatabaseAdapter) IntFunction(java.util.function.IntFunction) MethodSource(org.junit.jupiter.params.provider.MethodSource) WithGlobalStateContent(org.projectnessie.versioned.testworker.WithGlobalStateContent) Hash(org.projectnessie.versioned.Hash) Builder(org.projectnessie.versioned.persist.adapter.ImmutableCommitAttempt.Builder) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Set(java.util.Set) DatabaseAdapterMetrics(org.projectnessie.versioned.persist.adapter.spi.DatabaseAdapterMetrics) Key(org.projectnessie.versioned.Key) Collectors(java.util.stream.Collectors) ReferenceNotFoundException(org.projectnessie.versioned.ReferenceNotFoundException) KeyWithBytes(org.projectnessie.versioned.persist.adapter.KeyWithBytes) Metrics.globalRegistry(io.micrometer.core.instrument.Metrics.globalRegistry) Executors(java.util.concurrent.Executors) ByteString(com.google.protobuf.ByteString) TimeUnit(java.util.concurrent.TimeUnit) CountDownLatch(java.util.concurrent.CountDownLatch) BranchName(org.projectnessie.versioned.BranchName) ReferenceConflictException(org.projectnessie.versioned.ReferenceConflictException) List(java.util.List) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) Stream(java.util.stream.Stream) ImmutableCommitAttempt(org.projectnessie.versioned.persist.adapter.ImmutableCommitAttempt) ContentAndState(org.projectnessie.versioned.persist.adapter.ContentAndState) Entry(java.util.Map.Entry) Optional(java.util.Optional) SimpleStoreWorker(org.projectnessie.versioned.testworker.SimpleStoreWorker) ContentId(org.projectnessie.versioned.persist.adapter.ContentId) HashSet(java.util.HashSet) Set(java.util.Set) HashMap(java.util.HashMap) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ByteString(com.google.protobuf.ByteString) ArrayList(java.util.ArrayList) ByteString(com.google.protobuf.ByteString) Hash(org.projectnessie.versioned.Hash) WithGlobalStateContent(org.projectnessie.versioned.testworker.WithGlobalStateContent) CompletableFuture(java.util.concurrent.CompletableFuture) BranchName(org.projectnessie.versioned.BranchName) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) ReferenceRetryFailureException(org.projectnessie.versioned.ReferenceRetryFailureException) HashSet(java.util.HashSet) SimpleMeterRegistry(io.micrometer.core.instrument.simple.SimpleMeterRegistry) ImmutableCommitAttempt(org.projectnessie.versioned.persist.adapter.ImmutableCommitAttempt) CountDownLatch(java.util.concurrent.CountDownLatch) ReferenceRetryFailureException(org.projectnessie.versioned.ReferenceRetryFailureException) ReferenceNotFoundException(org.projectnessie.versioned.ReferenceNotFoundException) ReferenceConflictException(org.projectnessie.versioned.ReferenceConflictException) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) IntFunction(java.util.function.IntFunction) ExecutorService(java.util.concurrent.ExecutorService) ContentId(org.projectnessie.versioned.persist.adapter.ContentId) HashMap(java.util.HashMap) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) Key(org.projectnessie.versioned.Key) Builder(org.projectnessie.versioned.persist.adapter.ImmutableCommitAttempt.Builder) Metrics.globalRegistry(io.micrometer.core.instrument.Metrics.globalRegistry) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) MethodSource(org.junit.jupiter.params.provider.MethodSource)

Example 2 with CommitAttempt

use of org.projectnessie.versioned.persist.adapter.CommitAttempt in project nessie by projectnessie.

the class AbstractConcurrency method commitAndRecord.

private void commitAndRecord(Map<ContentId, ByteString> globalStates, Map<BranchName, Map<Key, ByteString>> onRefStates, BranchName branch, Builder commitAttempt) throws ReferenceConflictException, ReferenceNotFoundException {
    CommitAttempt c = commitAttempt.build();
    databaseAdapter.commit(c);
    globalStates.putAll(c.getGlobal());
    Map<Key, ByteString> onRef = onRefStates.computeIfAbsent(branch, b -> new ConcurrentHashMap<>());
    c.getPuts().forEach(kwb -> onRef.put(kwb.getKey(), kwb.getValue()));
}
Also used : ByteString(com.google.protobuf.ByteString) Key(org.projectnessie.versioned.Key) CommitAttempt(org.projectnessie.versioned.persist.adapter.CommitAttempt) ImmutableCommitAttempt(org.projectnessie.versioned.persist.adapter.ImmutableCommitAttempt)

Aggregations

ByteString (com.google.protobuf.ByteString)2 Key (org.projectnessie.versioned.Key)2 CommitAttempt (org.projectnessie.versioned.persist.adapter.CommitAttempt)2 ImmutableCommitAttempt (org.projectnessie.versioned.persist.adapter.ImmutableCommitAttempt)2 Metrics.globalRegistry (io.micrometer.core.instrument.Metrics.globalRegistry)1 SimpleMeterRegistry (io.micrometer.core.instrument.simple.SimpleMeterRegistry)1 ArrayList (java.util.ArrayList)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Entry (java.util.Map.Entry)1 Optional (java.util.Optional)1 Set (java.util.Set)1 CompletableFuture (java.util.concurrent.CompletableFuture)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 ExecutorService (java.util.concurrent.ExecutorService)1 Executors (java.util.concurrent.Executors)1 ThreadLocalRandom (java.util.concurrent.ThreadLocalRandom)1