Examples with CopyOnWriteArrayList java.util.concurrent.CopyOnWriteArrayList used on opensource projects

Example 66 with CopyOnWriteArrayList

use of java.util.concurrent.CopyOnWriteArrayList in project elasticsearch by elastic.

the class RemoteClusterConnectionTests method testFilterDiscoveredNodes.

public void testFilterDiscoveredNodes() throws Exception {
    List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
    try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.CURRENT);
        MockTransportService discoverableTransport = startTransport("discoverable_node", knownNodes, Version.CURRENT)) {
        DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
        DiscoveryNode discoverableNode = discoverableTransport.getLocalDiscoNode();
        knownNodes.add(seedTransport.getLocalDiscoNode());
        knownNodes.add(discoverableTransport.getLocalDiscoNode());
        DiscoveryNode rejectedNode = randomBoolean() ? seedNode : discoverableNode;
        Collections.shuffle(knownNodes, random());
        try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
            service.start();
            service.acceptIncomingRequests();
            try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> n.equals(rejectedNode) == false)) {
                updateSeedNodes(connection, Arrays.asList(seedNode));
                if (rejectedNode.equals(seedNode)) {
                    assertFalse(service.nodeConnected(seedNode));
                    assertTrue(service.nodeConnected(discoverableNode));
                } else {
                    assertTrue(service.nodeConnected(seedNode));
                    assertFalse(service.nodeConnected(discoverableNode));
                }
                assertTrue(connection.assertNoRunningConnections());
            }
        }
    }
}

Example 67 with CopyOnWriteArrayList

use of java.util.concurrent.CopyOnWriteArrayList in project elasticsearch by elastic.

the class RemoteClusterConnectionTests method testConnectWithIncompatibleTransports.

public void testConnectWithIncompatibleTransports() throws Exception {
    List<DiscoveryNode> knownNodes = new CopyOnWriteArrayList<>();
    try (MockTransportService seedTransport = startTransport("seed_node", knownNodes, Version.fromString("2.0.0"))) {
        DiscoveryNode seedNode = seedTransport.getLocalDiscoNode();
        knownNodes.add(seedTransport.getLocalDiscoNode());
        try (MockTransportService service = MockTransportService.createNewService(Settings.EMPTY, Version.CURRENT, threadPool, null)) {
            service.start();
            service.acceptIncomingRequests();
            try (RemoteClusterConnection connection = new RemoteClusterConnection(Settings.EMPTY, "test-cluster", Arrays.asList(seedNode), service, Integer.MAX_VALUE, n -> true)) {
                expectThrows(Exception.class, () -> updateSeedNodes(connection, Arrays.asList(seedNode)));
                assertFalse(service.nodeConnected(seedNode));
                assertTrue(connection.assertNoRunningConnections());
            }
        }
    }
}

Example 68 with CopyOnWriteArrayList

use of java.util.concurrent.CopyOnWriteArrayList in project elasticsearch by elastic.

the class TranslogTests method testConcurrentWriteViewsAndSnapshot.

/**
     * Tests that concurrent readers and writes maintain view and snapshot semantics
     */
public void testConcurrentWriteViewsAndSnapshot() throws Throwable {
    final Thread[] writers = new Thread[randomIntBetween(1, 10)];
    final Thread[] readers = new Thread[randomIntBetween(1, 10)];
    final int flushEveryOps = randomIntBetween(5, 100);
    // used to notify main thread that so many operations have been written so it can simulate a flush
    final AtomicReference<CountDownLatch> writtenOpsLatch = new AtomicReference<>(new CountDownLatch(0));
    final AtomicLong idGenerator = new AtomicLong();
    final CyclicBarrier barrier = new CyclicBarrier(writers.length + readers.length + 1);
    // a map of all written ops and their returned location.
    final Map<Translog.Operation, Translog.Location> writtenOps = ConcurrentCollections.newConcurrentMap();
    // a signal for all threads to stop
    final AtomicBoolean run = new AtomicBoolean(true);
    // any errors on threads
    final List<Exception> errors = new CopyOnWriteArrayList<>();
    logger.debug("using [{}] readers. [{}] writers. flushing every ~[{}] ops.", readers.length, writers.length, flushEveryOps);
    for (int i = 0; i < writers.length; i++) {
        final String threadName = "writer_" + i;
        final int threadId = i;
        writers[i] = new Thread(new AbstractRunnable() {

            @Override
            public void doRun() throws BrokenBarrierException, InterruptedException, IOException {
                barrier.await();
                int counter = 0;
                while (run.get()) {
                    long id = idGenerator.incrementAndGet();
                    final Translog.Operation op;
                    final Translog.Operation.Type type = Translog.Operation.Type.values()[((int) (id % Translog.Operation.Type.values().length))];
                    switch(type) {
                        case CREATE:
                        case INDEX:
                            op = new Translog.Index("type", "" + id, new byte[] { (byte) id });
                            break;
                        case DELETE:
                            op = new Translog.Delete(newUid("" + id));
                            break;
                        case NO_OP:
                            op = new Translog.NoOp(id, id, Long.toString(id));
                            break;
                        default:
                            throw new AssertionError("unsupported operation type [" + type + "]");
                    }
                    Translog.Location location = translog.add(op);
                    Translog.Location existing = writtenOps.put(op, location);
                    if (existing != null) {
                        fail("duplicate op [" + op + "], old entry at " + location);
                    }
                    if (id % writers.length == threadId) {
                        translog.ensureSynced(location);
                    }
                    writtenOpsLatch.get().countDown();
                    counter++;
                }
                logger.debug("--> [{}] done. wrote [{}] ops.", threadName, counter);
            }

            @Override
            public void onFailure(Exception e) {
                logger.error((Supplier<?>) () -> new ParameterizedMessage("--> writer [{}] had an error", threadName), e);
                errors.add(e);
            }
        }, threadName);
        writers[i].start();
    }
    for (int i = 0; i < readers.length; i++) {
        final String threadId = "reader_" + i;
        readers[i] = new Thread(new AbstractRunnable() {

            Translog.View view = null;

            Set<Translog.Operation> writtenOpsAtView;

            @Override
            public void onFailure(Exception e) {
                logger.error((Supplier<?>) () -> new ParameterizedMessage("--> reader [{}] had an error", threadId), e);
                errors.add(e);
                try {
                    closeView();
                } catch (IOException inner) {
                    inner.addSuppressed(e);
                    logger.error("unexpected error while closing view, after failure", inner);
                }
            }

            void closeView() throws IOException {
                if (view != null) {
                    view.close();
                }
            }

            void newView() throws IOException {
                closeView();
                view = translog.newView();
                // captures the currently written ops so we know what to expect from the view
                writtenOpsAtView = new HashSet<>(writtenOps.keySet());
                logger.debug("--> [{}] opened view from [{}]", threadId, view.minTranslogGeneration());
            }

            @Override
            protected void doRun() throws Exception {
                barrier.await();
                int iter = 0;
                while (run.get()) {
                    if (iter++ % 10 == 0) {
                        newView();
                    }
                    // captures al views that are written since the view was created (with a small caveat see bellow)
                    // these are what we expect the snapshot to return (and potentially some more).
                    Set<Translog.Operation> expectedOps = new HashSet<>(writtenOps.keySet());
                    expectedOps.removeAll(writtenOpsAtView);
                    Translog.Snapshot snapshot = view.snapshot();
                    Translog.Operation op;
                    while ((op = snapshot.next()) != null) {
                        expectedOps.remove(op);
                    }
                    if (expectedOps.isEmpty() == false) {
                        StringBuilder missed = new StringBuilder("missed ").append(expectedOps.size()).append(" operations");
                        boolean failed = false;
                        for (Translog.Operation expectedOp : expectedOps) {
                            final Translog.Location loc = writtenOps.get(expectedOp);
                            if (loc.generation < view.minTranslogGeneration()) {
                                // may yet be available in writtenOpsAtView, meaning we will erroneously expect them
                                continue;
                            }
                            failed = true;
                            missed.append("\n --> [").append(expectedOp).append("] written at ").append(loc);
                        }
                        if (failed) {
                            fail(missed.toString());
                        }
                    }
                    // slow down things a bit and spread out testing..
                    writtenOpsLatch.get().await(200, TimeUnit.MILLISECONDS);
                }
                closeView();
                logger.debug("--> [{}] done. tested [{}] snapshots", threadId, iter);
            }
        }, threadId);
        readers[i].start();
    }
    barrier.await();
    try {
        for (int iterations = scaledRandomIntBetween(10, 200); iterations > 0 && errors.isEmpty(); iterations--) {
            writtenOpsLatch.set(new CountDownLatch(flushEveryOps));
            while (writtenOpsLatch.get().await(200, TimeUnit.MILLISECONDS) == false) {
                if (errors.size() > 0) {
                    break;
                }
            }
            translog.commit();
        }
    } finally {
        run.set(false);
        logger.debug("--> waiting for threads to stop");
        for (Thread thread : writers) {
            thread.join();
        }
        for (Thread thread : readers) {
            thread.join();
        }
        if (errors.size() > 0) {
            Throwable e = errors.get(0);
            for (Throwable suppress : errors.subList(1, errors.size())) {
                e.addSuppressed(suppress);
            }
            throw e;
        }
        logger.info("--> test done. total ops written [{}]", writtenOps.size());
    }
}

Example 69 with CopyOnWriteArrayList

use of java.util.concurrent.CopyOnWriteArrayList in project elasticsearch by elastic.

the class RefCountedTests method testMultiThreaded.

public void testMultiThreaded() throws InterruptedException {
    final MyRefCounted counted = new MyRefCounted();
    Thread[] threads = new Thread[randomIntBetween(2, 5)];
    final CountDownLatch latch = new CountDownLatch(1);
    final CopyOnWriteArrayList<Exception> exceptions = new CopyOnWriteArrayList<>();
    for (int i = 0; i < threads.length; i++) {
        threads[i] = new Thread() {

            @Override
            public void run() {
                try {
                    latch.await();
                    for (int j = 0; j < 10000; j++) {
                        counted.incRef();
                        try {
                            counted.ensureOpen();
                        } finally {
                            counted.decRef();
                        }
                    }
                } catch (Exception e) {
                    exceptions.add(e);
                }
            }
        };
        threads[i].start();
    }
    latch.countDown();
    for (int i = 0; i < threads.length; i++) {
        threads[i].join();
    }
    counted.decRef();
    try {
        counted.ensureOpen();
        fail("expected to be closed");
    } catch (AlreadyClosedException ex) {
        assertThat(ex.getMessage(), equalTo("closed"));
    }
    assertThat(counted.refCount(), is(0));
    assertThat(exceptions, Matchers.emptyIterable());
}

Example 70 with CopyOnWriteArrayList

use of java.util.concurrent.CopyOnWriteArrayList in project elasticsearch by elastic.

the class IndexStatsIT method testConcurrentIndexingAndStatsRequests.

/**
     * Test that we can safely concurrently index and get stats. This test was inspired by a serialization issue that arose due to a race
     * getting doc stats during heavy indexing. The race could lead to deleted docs being negative which would then be serialized as a
     * variable-length long. Since serialization of negative longs using a variable-length format was unsupported
     * ({@link org.elasticsearch.common.io.stream.StreamOutput#writeVLong(long)}), the stream would become corrupted. Here, we want to test
     * that we can continue to get stats while indexing.
     */
public void testConcurrentIndexingAndStatsRequests() throws BrokenBarrierException, InterruptedException, ExecutionException {
    final AtomicInteger idGenerator = new AtomicInteger();
    final int numberOfIndexingThreads = Runtime.getRuntime().availableProcessors();
    final int numberOfStatsThreads = 4 * numberOfIndexingThreads;
    final CyclicBarrier barrier = new CyclicBarrier(1 + numberOfIndexingThreads + numberOfStatsThreads);
    final AtomicBoolean stop = new AtomicBoolean();
    final List<Thread> threads = new ArrayList<>(numberOfIndexingThreads + numberOfIndexingThreads);
    final CountDownLatch latch = new CountDownLatch(1);
    final AtomicBoolean failed = new AtomicBoolean();
    final AtomicReference<List<ShardOperationFailedException>> shardFailures = new AtomicReference<>(new CopyOnWriteArrayList<>());
    final AtomicReference<List<Exception>> executionFailures = new AtomicReference<>(new CopyOnWriteArrayList<>());
    // increasing the number of shards increases the number of chances any one stats request will hit a race
    final CreateIndexRequest createIndexRequest = new CreateIndexRequest("test", Settings.builder().put("index.number_of_shards", 10).build());
    client().admin().indices().create(createIndexRequest).get();
    // start threads that will index concurrently with stats requests
    for (int i = 0; i < numberOfIndexingThreads; i++) {
        final Thread thread = new Thread(() -> {
            try {
                barrier.await();
            } catch (final BrokenBarrierException | InterruptedException e) {
                failed.set(true);
                executionFailures.get().add(e);
                latch.countDown();
            }
            while (!stop.get()) {
                final String id = Integer.toString(idGenerator.incrementAndGet());
                final IndexResponse response = client().prepareIndex("test", "type", id).setSource("{}", XContentType.JSON).get();
                assertThat(response.getResult(), equalTo(DocWriteResponse.Result.CREATED));
            }
        });
        thread.setName("indexing-" + i);
        threads.add(thread);
        thread.start();
    }
    // start threads that will get stats concurrently with indexing
    for (int i = 0; i < numberOfStatsThreads; i++) {
        final Thread thread = new Thread(() -> {
            try {
                barrier.await();
            } catch (final BrokenBarrierException | InterruptedException e) {
                failed.set(true);
                executionFailures.get().add(e);
                latch.countDown();
            }
            final IndicesStatsRequest request = new IndicesStatsRequest();
            request.all();
            request.indices(new String[0]);
            while (!stop.get()) {
                try {
                    final IndicesStatsResponse response = client().admin().indices().stats(request).get();
                    if (response.getFailedShards() > 0) {
                        failed.set(true);
                        shardFailures.get().addAll(Arrays.asList(response.getShardFailures()));
                        latch.countDown();
                    }
                } catch (final ExecutionException | InterruptedException e) {
                    failed.set(true);
                    executionFailures.get().add(e);
                    latch.countDown();
                }
            }
        });
        thread.setName("stats-" + i);
        threads.add(thread);
        thread.start();
    }
    // release the hounds
    barrier.await();
    // wait for a failure, or for fifteen seconds to elapse
    latch.await(15, TimeUnit.SECONDS);
    // stop all threads and wait for them to complete
    stop.set(true);
    for (final Thread thread : threads) {
        thread.join();
    }
    assertThat(shardFailures.get(), emptyCollectionOf(ShardOperationFailedException.class));
    assertThat(executionFailures.get(), emptyCollectionOf(Exception.class));
}