Examples with ConcurrentLinkedDeque java.util.concurrent.ConcurrentLinkedDeque used on opensource projects

Example 66 with ConcurrentLinkedDeque

use of java.util.concurrent.ConcurrentLinkedDeque in project neo4j by neo4j.

the class DenseNodeConcurrencyIT method shouldCreateAndDeleteRelationshipsConcurrently.

/**
 * @param multipleDenseNodes if {@code true} then multiple subject nodes are created and will get relationships created between each other during the test,
 * otherwise only one dense node is the test subject and other nodes are just small created nodes.
 * @param startAsDense if {@code true} then the subject node(s) will start as dense and has lots of relationships of the initial type,
 * otherwise the node(s) will start as sparse with only a few relationships.
 * @param multipleOperationsInOneTx if {@code true} then each transaction have a chance to do multiple operations,
 * otherwise each transaction only performs one operation.
 * @param multipleTypes if {@code true} then relationships will be either of several types, otherwise only the same type.
 * @param operationWeights chances of each type of operation happening in the test.
 */
@MethodSource("permutations")
@ParameterizedTest(name = "multipleDenseNodes:{0}, startAsDense:{1}, multipleOpsPerTx:{2}, multipleTypes:{3}, opWeights:{4}")
void shouldCreateAndDeleteRelationshipsConcurrently(boolean multipleDenseNodes, boolean startAsDense, boolean multipleOperationsInOneTx, boolean multipleTypes, Map<WorkType, Integer> operationWeights) {
    // given
    Map<Long, Set<Relationship>> relationships = new ConcurrentHashMap<>();
    Set<Relationship> allRelationships = newKeySet();
    Set<Long> initialDenseNodes = new HashSet<>(createInitialNodes(multipleDenseNodes, startAsDense, relationships));
    Set<Long> denseNodeIds = ConcurrentHashMap.newKeySet();
    denseNodeIds.addAll(initialDenseNodes);
    relationships.forEach((nodeId, rels) -> allRelationships.addAll(rels));
    // when
    Queue<WorkTask> workQueue = new ConcurrentLinkedDeque<>(createWork(operationWeights));
    Race race = new Race().withFailureAction(t -> workQueue.clear());
    int numWorkers = Runtime.getRuntime().availableProcessors();
    List<RelationshipType> typesList = new ArrayList<>();
    typesList.add(INITIAL_DENSE_NODE_TYPE);
    if (multipleTypes) {
        typesList.add(withName("a"));
        typesList.add(withName("b"));
    }
    RelationshipType[] types = typesList.toArray(new RelationshipType[0]);
    AtomicInteger numDeadlocks = new AtomicInteger();
    race.addContestants(numWorkers, throwing(() -> {
        WorkTask work;
        while ((work = workQueue.poll()) != null) {
            // Construct all the tasks that this transaction will perform
            List<WorkTask> txTasks = new ArrayList<>();
            txTasks.add(work);
            while (multipleOperationsInOneTx && random.nextBoolean() && (work = workQueue.poll()) != null) {
                txTasks.add(work);
            }
            // Try to perform those operations, if we fail on dead-lock then simply retry all those operations until we don't
            boolean retry;
            int numRetries = 0;
            do {
                // Intermediary state of created/deleted relationships to update or relationships mirror with upon success of the transaction
                Map<Long, TxNodeChanges> txCreated = new HashMap<>();
                Map<Long, TxNodeChanges> txDeleted = new HashMap<>();
                try (Transaction tx = database.beginTx()) {
                    for (WorkTask task : txTasks) {
                        task.perform(tx, denseNodeIds, random.among(types), relationships, allRelationships, random, txCreated, txDeleted);
                    }
                    tx.commit();
                    retry = false;
                    // Now on success update each node's relationship mirror
                    txCreated.forEach((nodeId, changes) -> relationships.get(nodeId).addAll(changes.relationships));
                    txDeleted.forEach((nodeId, changes) -> relationships.get(nodeId).removeAll(changes.relationships));
                    // Finally update the global relationships map afterwards so that no deleter is able to spot them before we've updated the mirrors
                    txCreated.forEach((nodeId, changes) -> allRelationships.addAll(changes.relationships));
                } catch (TransientTransactionFailureException e) {
                    retry = true;
                    numRetries++;
                    allRelationships.addAll(txDeleted.values().stream().flatMap(change -> change.relationships.stream()).collect(Collectors.toSet()));
                    denseNodeIds.addAll(txDeleted.values().stream().filter(change -> change.node).map(change -> change.id).collect(Collectors.toList()));
                    numDeadlocks.incrementAndGet();
                    // Random back-off after deadlock
                    Thread.sleep(random.nextInt(1, 10 * numRetries));
                }
            } while (retry);
        }
    }), 1);
    race.goUnchecked();
    // then
    Set<Long> deletedDenseNodes = new HashSet<>(initialDenseNodes);
    deletedDenseNodes.removeAll(denseNodeIds);
    assertDeletedNodes(deletedDenseNodes);
    for (long denseNodeId : denseNodeIds) {
        assertRelationshipsAndDegrees(denseNodeId, relationships.get(denseNodeId));
    }
    assertThat(numDeadlocks.get()).isLessThan(NUM_TASKS / 10);
}

Example 67 with ConcurrentLinkedDeque

use of java.util.concurrent.ConcurrentLinkedDeque in project kafka by apache.

the class MetricsTest method testConcurrentReadUpdate.

/**
 * Verifies that concurrent sensor add, remove, updates and read don't result
 * in errors or deadlock.
 */
@Test
public void testConcurrentReadUpdate() throws Exception {
    final Random random = new Random();
    final Deque<Sensor> sensors = new ConcurrentLinkedDeque<>();
    metrics = new Metrics(new MockTime(10));
    SensorCreator sensorCreator = new SensorCreator(metrics);
    final AtomicBoolean alive = new AtomicBoolean(true);
    executorService = Executors.newSingleThreadExecutor();
    executorService.submit(new ConcurrentMetricOperation(alive, "record", () -> sensors.forEach(sensor -> sensor.record(random.nextInt(10000)))));
    for (int i = 0; i < 10000; i++) {
        if (sensors.size() > 5) {
            Sensor sensor = random.nextBoolean() ? sensors.removeFirst() : sensors.removeLast();
            metrics.removeSensor(sensor.name());
        }
        StatType statType = StatType.forId(random.nextInt(StatType.values().length));
        sensors.add(sensorCreator.createSensor(statType, i));
        for (Sensor sensor : sensors) {
            for (KafkaMetric metric : sensor.metrics()) {
                assertNotNull(metric.metricValue(), "Invalid metric value");
            }
        }
    }
    alive.set(false);
}

Example 68 with ConcurrentLinkedDeque

use of java.util.concurrent.ConcurrentLinkedDeque in project flink by apache.

the class StreamCollector method collect.

/**
 * @return A future that contains all the elements of the DataStream which completes when all
 *     elements have been processed.
 */
public <IN> CompletableFuture<Collection<IN>> collect(DataStream<IN> stream) {
    final long id = counter.getAndIncrement();
    ids.add(id);
    int parallelism = stream.getParallelism();
    if (parallelism == ExecutionConfig.PARALLELISM_DEFAULT) {
        parallelism = stream.getExecutionEnvironment().getParallelism();
    }
    CountDownLatch latch = new CountDownLatch(parallelism);
    latches.put(id, latch);
    Queue<IN> results = new ConcurrentLinkedDeque<>();
    resultQueues.put(id, results);
    stream.addSink(new CollectingSink<>(id));
    return CompletableFuture.runAsync(() -> {
        try {
            latch.await();
        } catch (InterruptedException e) {
            throw new RuntimeException("Failed to collect results");
        }
    }).thenApply(ignore -> results);
}

Example 69 with ConcurrentLinkedDeque

use of java.util.concurrent.ConcurrentLinkedDeque in project flink by apache.

the class StreamCollectorExtension method collect.

/**
 * @return A future that contains all the elements of the DataStream which completes when all
 *     elements have been processed.
 */
public <IN> CompletableFuture<Collection<IN>> collect(DataStream<IN> stream) {
    final long id = counter.getAndIncrement();
    ids.add(id);
    int parallelism = stream.getParallelism();
    if (parallelism == ExecutionConfig.PARALLELISM_DEFAULT) {
        parallelism = stream.getExecutionEnvironment().getParallelism();
    }
    CountDownLatch latch = new CountDownLatch(parallelism);
    latches.put(id, latch);
    Queue<IN> results = new ConcurrentLinkedDeque<>();
    resultQueues.put(id, results);
    stream.addSink(new StreamCollectorExtension.CollectingSink<>(id));
    return CompletableFuture.runAsync(() -> {
        try {
            latch.await();
        } catch (InterruptedException e) {
            throw new RuntimeException("Failed to collect results");
        }
    }).thenApply(ignore -> results);
}

Example 70 with ConcurrentLinkedDeque

use of java.util.concurrent.ConcurrentLinkedDeque in project alluxio by Alluxio.

the class BlockMasterWorkerServiceHandlerTest method registerLeaseExpired.

@Test
public void registerLeaseExpired() {
    long workerId = mBlockMaster.getWorkerId(NET_ADDRESS_1);
    // Prepare LocationBlockIdListEntry objects
    BlockStoreLocation loc = new BlockStoreLocation("MEM", 0);
    BlockStoreLocationProto locationProto = BlockStoreLocationProto.newBuilder().setTierAlias(loc.tierAlias()).setMediumType(loc.mediumType()).build();
    BlockIdList blockIdList1 = BlockIdList.newBuilder().addAllBlockId(ImmutableList.of(1L, 2L)).build();
    LocationBlockIdListEntry listEntry1 = LocationBlockIdListEntry.newBuilder().setKey(locationProto).setValue(blockIdList1).build();
    // Prepare a lease
    GetRegisterLeasePRequest leaseRequest = GetRegisterLeasePRequest.newBuilder().setWorkerId(workerId).setBlockCount(blockIdList1.getBlockIdCount()).build();
    Optional<RegisterLease> lease = mBlockMaster.tryAcquireRegisterLease(leaseRequest);
    assertTrue(lease.isPresent());
    // Sleep for a while so that the lease expires
    SleepUtils.sleepMs(5000);
    // The lease is recycled and taken away
    GetRegisterLeasePRequest newLeaseRequest = GetRegisterLeasePRequest.newBuilder().setWorkerId(workerId + 1).setBlockCount(blockIdList1.getBlockIdCount()).build();
    Optional<RegisterLease> newLease = mBlockMaster.tryAcquireRegisterLease(newLeaseRequest);
    assertTrue(newLease.isPresent());
    RegisterWorkerPRequest request = RegisterWorkerPRequest.newBuilder().setWorkerId(workerId).addStorageTiers("MEM").putTotalBytesOnTiers("MEM", 1000L).putUsedBytesOnTiers("MEM", 0L).setOptions(RegisterWorkerPOptions.getDefaultInstance()).addCurrentBlocks(listEntry1).build();
    // Noop response observer
    Queue<Throwable> errors = new ConcurrentLinkedDeque<>();
    StreamObserver<RegisterWorkerPResponse> noopResponseObserver = new StreamObserver<RegisterWorkerPResponse>() {

        @Override
        public void onNext(RegisterWorkerPResponse response) {
        }

        @Override
        public void onError(Throwable t) {
            errors.offer(t);
        }

        @Override
        public void onCompleted() {
        }
    };
    mHandler.registerWorker(request, noopResponseObserver);
    assertEquals(1, errors.size());
    Throwable t = errors.poll();
    Assert.assertThat(t.getMessage(), containsString("does not have a lease or the lease has expired."));
    mBlockMaster.releaseRegisterLease(workerId + 1);
}