Search in sources :

Example 1 with ConcurrentHashMap.newKeySet

use of java.util.concurrent.ConcurrentHashMap.newKeySet in project neo4j by neo4j.

the class DenseNodeConcurrencyIT method shouldCreateAndDeleteRelationshipsConcurrently.

/**
 * @param multipleDenseNodes if {@code true} then multiple subject nodes are created and will get relationships created between each other during the test,
 * otherwise only one dense node is the test subject and other nodes are just small created nodes.
 * @param startAsDense if {@code true} then the subject node(s) will start as dense and has lots of relationships of the initial type,
 * otherwise the node(s) will start as sparse with only a few relationships.
 * @param multipleOperationsInOneTx if {@code true} then each transaction have a chance to do multiple operations,
 * otherwise each transaction only performs one operation.
 * @param multipleTypes if {@code true} then relationships will be either of several types, otherwise only the same type.
 * @param operationWeights chances of each type of operation happening in the test.
 */
@MethodSource("permutations")
@ParameterizedTest(name = "multipleDenseNodes:{0}, startAsDense:{1}, multipleOpsPerTx:{2}, multipleTypes:{3}, opWeights:{4}")
void shouldCreateAndDeleteRelationshipsConcurrently(boolean multipleDenseNodes, boolean startAsDense, boolean multipleOperationsInOneTx, boolean multipleTypes, Map<WorkType, Integer> operationWeights) {
    // given
    Map<Long, Set<Relationship>> relationships = new ConcurrentHashMap<>();
    Set<Relationship> allRelationships = newKeySet();
    Set<Long> initialDenseNodes = new HashSet<>(createInitialNodes(multipleDenseNodes, startAsDense, relationships));
    Set<Long> denseNodeIds = ConcurrentHashMap.newKeySet();
    denseNodeIds.addAll(initialDenseNodes);
    relationships.forEach((nodeId, rels) -> allRelationships.addAll(rels));
    // when
    Queue<WorkTask> workQueue = new ConcurrentLinkedDeque<>(createWork(operationWeights));
    Race race = new Race().withFailureAction(t -> workQueue.clear());
    int numWorkers = Runtime.getRuntime().availableProcessors();
    List<RelationshipType> typesList = new ArrayList<>();
    typesList.add(INITIAL_DENSE_NODE_TYPE);
    if (multipleTypes) {
        typesList.add(withName("a"));
        typesList.add(withName("b"));
    }
    RelationshipType[] types = typesList.toArray(new RelationshipType[0]);
    AtomicInteger numDeadlocks = new AtomicInteger();
    race.addContestants(numWorkers, throwing(() -> {
        WorkTask work;
        while ((work = workQueue.poll()) != null) {
            // Construct all the tasks that this transaction will perform
            List<WorkTask> txTasks = new ArrayList<>();
            txTasks.add(work);
            while (multipleOperationsInOneTx && random.nextBoolean() && (work = workQueue.poll()) != null) {
                txTasks.add(work);
            }
            // Try to perform those operations, if we fail on dead-lock then simply retry all those operations until we don't
            boolean retry;
            int numRetries = 0;
            do {
                // Intermediary state of created/deleted relationships to update or relationships mirror with upon success of the transaction
                Map<Long, TxNodeChanges> txCreated = new HashMap<>();
                Map<Long, TxNodeChanges> txDeleted = new HashMap<>();
                try (Transaction tx = database.beginTx()) {
                    for (WorkTask task : txTasks) {
                        task.perform(tx, denseNodeIds, random.among(types), relationships, allRelationships, random, txCreated, txDeleted);
                    }
                    tx.commit();
                    retry = false;
                    // Now on success update each node's relationship mirror
                    txCreated.forEach((nodeId, changes) -> relationships.get(nodeId).addAll(changes.relationships));
                    txDeleted.forEach((nodeId, changes) -> relationships.get(nodeId).removeAll(changes.relationships));
                    // Finally update the global relationships map afterwards so that no deleter is able to spot them before we've updated the mirrors
                    txCreated.forEach((nodeId, changes) -> allRelationships.addAll(changes.relationships));
                } catch (TransientTransactionFailureException e) {
                    retry = true;
                    numRetries++;
                    allRelationships.addAll(txDeleted.values().stream().flatMap(change -> change.relationships.stream()).collect(Collectors.toSet()));
                    denseNodeIds.addAll(txDeleted.values().stream().filter(change -> change.node).map(change -> change.id).collect(Collectors.toList()));
                    numDeadlocks.incrementAndGet();
                    // Random back-off after deadlock
                    Thread.sleep(random.nextInt(1, 10 * numRetries));
                }
            } while (retry);
        }
    }), 1);
    race.goUnchecked();
    // then
    Set<Long> deletedDenseNodes = new HashSet<>(initialDenseNodes);
    deletedDenseNodes.removeAll(denseNodeIds);
    assertDeletedNodes(deletedDenseNodes);
    for (long denseNodeId : denseNodeIds) {
        assertRelationshipsAndDegrees(denseNodeId, relationships.get(denseNodeId));
    }
    assertThat(numDeadlocks.get()).isLessThan(NUM_TASKS / 10);
}
Also used : BeforeEach(org.junit.jupiter.api.BeforeEach) Record.isNull(org.neo4j.kernel.impl.store.record.Record.isNull) Arrays(java.util.Arrays) MapUtil(org.neo4j.internal.helpers.collection.MapUtil) RandomExtension(org.neo4j.test.extension.RandomExtension) Status(org.neo4j.kernel.api.exceptions.Status) Assertions.assertThat(org.assertj.core.api.Assertions.assertThat) TimeoutException(java.util.concurrent.TimeoutException) Config(org.neo4j.configuration.Config) InternalTransaction(org.neo4j.kernel.impl.coreapi.InternalTransaction) DatabaseLayout(org.neo4j.io.layout.DatabaseLayout) AfterAll(org.junit.jupiter.api.AfterAll) ImpermanentDbmsExtension(org.neo4j.test.extension.ImpermanentDbmsExtension) Future(java.util.concurrent.Future) TestInstance(org.junit.jupiter.api.TestInstance) ExtendWith(org.junit.jupiter.api.extension.ExtendWith) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) RandomRule(org.neo4j.test.rule.RandomRule) Map(java.util.Map) TransactionImpl(org.neo4j.kernel.impl.coreapi.TransactionImpl) RelationshipEntity(org.neo4j.kernel.impl.core.RelationshipEntity) DeadlockDetectedException(org.neo4j.kernel.DeadlockDetectedException) ConcurrentHashMap.newKeySet(java.util.concurrent.ConcurrentHashMap.newKeySet) MethodSource(org.junit.jupiter.params.provider.MethodSource) Record(org.neo4j.kernel.impl.store.record.Record) OtherThreadExecutor.command(org.neo4j.test.OtherThreadExecutor.command) Predicate(java.util.function.Predicate) UncloseableDelegatingFileSystemAbstraction(org.neo4j.io.fs.UncloseableDelegatingFileSystemAbstraction) Collection(java.util.Collection) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) TEST2(org.neo4j.kernel.impl.MyRelTypes.TEST2) Set(java.util.Set) RelationshipType.withName(org.neo4j.graphdb.RelationshipType.withName) Arguments(org.junit.jupiter.params.provider.Arguments) Collectors(java.util.stream.Collectors) String.format(java.lang.String.format) GraphDatabaseAPI(org.neo4j.kernel.internal.GraphDatabaseAPI) Test(org.junit.jupiter.api.Test) UncheckedIOException(java.io.UncheckedIOException) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Stream(java.util.stream.Stream) Queue(java.util.Queue) Race.throwing(org.neo4j.test.Race.throwing) DatabaseManagementService(org.neo4j.dbms.api.DatabaseManagementService) ConsistencyCheckService(org.neo4j.consistency.ConsistencyCheckService) TEST(org.neo4j.kernel.impl.MyRelTypes.TEST) NONE(org.neo4j.internal.helpers.progress.ProgressMonitorFactory.NONE) GraphDatabaseSettings(org.neo4j.configuration.GraphDatabaseSettings) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) Function(java.util.function.Function) ArrayList(java.util.ArrayList) ExtensionCallback(org.neo4j.test.extension.ExtensionCallback) HashSet(java.util.HashSet) TestDatabaseManagementServiceBuilder(org.neo4j.test.TestDatabaseManagementServiceBuilder) Description(org.assertj.core.description.Description) Charset(java.nio.charset.Charset) Inject(org.neo4j.test.extension.Inject) Assertions.assertThatThrownBy(org.assertj.core.api.Assertions.assertThatThrownBy) Iterables(org.neo4j.internal.helpers.collection.Iterables) StreamSupport(java.util.stream.StreamSupport) Arguments.arguments(org.junit.jupiter.params.provider.Arguments.arguments) DependencyResolver(org.neo4j.common.DependencyResolver) OtherThreadExecutor(org.neo4j.test.OtherThreadExecutor) ValueSource(org.junit.jupiter.params.provider.ValueSource) KernelTransactionImplementation(org.neo4j.kernel.impl.api.KernelTransactionImplementation) NullLogProvider.nullLogProvider(org.neo4j.logging.NullLogProvider.nullLogProvider) Barrier(org.neo4j.test.Barrier) PER_CLASS(org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS) IOException(java.io.IOException) ConcurrentLinkedDeque(java.util.concurrent.ConcurrentLinkedDeque) NodeEntity(org.neo4j.kernel.impl.core.NodeEntity) ExecutionException(java.util.concurrent.ExecutionException) Consumer(java.util.function.Consumer) ConsistencyCheckIncompleteException(org.neo4j.consistency.checking.full.ConsistencyCheckIncompleteException) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest) BufferedReader(java.io.BufferedReader) Collections(java.util.Collections) ConsistencyFlags(org.neo4j.consistency.checking.full.ConsistencyFlags) Race(org.neo4j.test.Race) FileSystemAbstraction(org.neo4j.io.fs.FileSystemAbstraction) ConcurrentHashMap.newKeySet(java.util.concurrent.ConcurrentHashMap.newKeySet) Set(java.util.Set) HashSet(java.util.HashSet) ArrayList(java.util.ArrayList) Race(org.neo4j.test.Race) List(java.util.List) ArrayList(java.util.ArrayList) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashSet(java.util.HashSet) ConcurrentLinkedDeque(java.util.concurrent.ConcurrentLinkedDeque) InternalTransaction(org.neo4j.kernel.impl.coreapi.InternalTransaction) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) Map(java.util.Map) ConcurrentHashMap(java.util.concurrent.ConcurrentHashMap) HashMap(java.util.HashMap) MethodSource(org.junit.jupiter.params.provider.MethodSource) ParameterizedTest(org.junit.jupiter.params.ParameterizedTest)

Aggregations

BufferedReader (java.io.BufferedReader)1 IOException (java.io.IOException)1 UncheckedIOException (java.io.UncheckedIOException)1 String.format (java.lang.String.format)1 Charset (java.nio.charset.Charset)1 ArrayList (java.util.ArrayList)1 Arrays (java.util.Arrays)1 Collection (java.util.Collection)1 Collections (java.util.Collections)1 HashMap (java.util.HashMap)1 HashSet (java.util.HashSet)1 List (java.util.List)1 Map (java.util.Map)1 Queue (java.util.Queue)1 Set (java.util.Set)1 ConcurrentHashMap (java.util.concurrent.ConcurrentHashMap)1 ConcurrentHashMap.newKeySet (java.util.concurrent.ConcurrentHashMap.newKeySet)1 ConcurrentLinkedDeque (java.util.concurrent.ConcurrentLinkedDeque)1 CountDownLatch (java.util.concurrent.CountDownLatch)1 ExecutionException (java.util.concurrent.ExecutionException)1