Examples with CopyOnWriteArrayList java.util.concurrent.CopyOnWriteArrayList used on opensource projects

Example 71 with CopyOnWriteArrayList

use of java.util.concurrent.CopyOnWriteArrayList in project elasticsearch by elastic.

the class ConcurrentDynamicTemplateIT method testConcurrentDynamicMapping.

// see #3544
public void testConcurrentDynamicMapping() throws Exception {
    final String fieldName = "field";
    final String mapping = "{ \"" + mappingType + "\": {" + "\"dynamic_templates\": [" + "{ \"" + fieldName + "\": {" + "\"path_match\": \"*\"," + "\"mapping\": {" + "\"type\": \"text\"," + "\"store\": true," + "\"analyzer\": \"whitespace\" } } } ] } }";
    // The 'fieldNames' array is used to help with retrieval of index terms
    // after testing
    int iters = scaledRandomIntBetween(5, 15);
    for (int i = 0; i < iters; i++) {
        cluster().wipeIndices("test");
        assertAcked(prepareCreate("test").addMapping(mappingType, mapping, XContentType.JSON));
        int numDocs = scaledRandomIntBetween(10, 100);
        final CountDownLatch latch = new CountDownLatch(numDocs);
        final List<Throwable> throwable = new CopyOnWriteArrayList<>();
        int currentID = 0;
        for (int j = 0; j < numDocs; j++) {
            Map<String, Object> source = new HashMap<>();
            source.put(fieldName, "test-user");
            client().prepareIndex("test", mappingType, Integer.toString(currentID++)).setSource(source).execute(new ActionListener<IndexResponse>() {

                @Override
                public void onResponse(IndexResponse response) {
                    latch.countDown();
                }

                @Override
                public void onFailure(Exception e) {
                    throwable.add(e);
                    latch.countDown();
                }
            });
        }
        latch.await();
        assertThat(throwable, emptyIterable());
        refresh();
        assertHitCount(client().prepareSearch("test").setQuery(QueryBuilders.matchQuery(fieldName, "test-user")).get(), numDocs);
        assertHitCount(client().prepareSearch("test").setQuery(QueryBuilders.matchQuery(fieldName, "test user")).get(), 0);
    }
}

Example 72 with CopyOnWriteArrayList

use of java.util.concurrent.CopyOnWriteArrayList in project elasticsearch by elastic.

the class SearchWhileRelocatingIT method testSearchAndRelocateConcurrently.

private void testSearchAndRelocateConcurrently(final int numberOfReplicas) throws Exception {
    final int numShards = between(1, 20);
    client().admin().indices().prepareCreate("test").setSettings(Settings.builder().put("index.number_of_shards", numShards).put("index.number_of_replicas", numberOfReplicas)).addMapping("type", "loc", "type=geo_point", "test", "type=text").execute().actionGet();
    ensureGreen();
    List<IndexRequestBuilder> indexBuilders = new ArrayList<>();
    final int numDocs = between(10, 20);
    for (int i = 0; i < numDocs; i++) {
        indexBuilders.add(client().prepareIndex("test", "type", Integer.toString(i)).setSource(jsonBuilder().startObject().field("test", "value").startObject("loc").field("lat", 11).field("lon", 21).endObject().endObject()));
    }
    indexRandom(true, indexBuilders.toArray(new IndexRequestBuilder[indexBuilders.size()]));
    assertHitCount(client().prepareSearch().get(), (numDocs));
    final int numIters = scaledRandomIntBetween(5, 20);
    for (int i = 0; i < numIters; i++) {
        final AtomicBoolean stop = new AtomicBoolean(false);
        final List<String> nonCriticalExceptions = new CopyOnWriteArrayList<>();
        Thread[] threads = new Thread[scaledRandomIntBetween(1, 3)];
        for (int j = 0; j < threads.length; j++) {
            threads[j] = new Thread() {

                @Override
                public void run() {
                    try {
                        while (!stop.get()) {
                            SearchResponse sr = client().prepareSearch().setSize(numDocs).get();
                            if (sr.getHits().getTotalHits() != numDocs) {
                                // request comes in. It's a small window but a known limitation.
                                if (sr.getTotalShards() != sr.getSuccessfulShards() && sr.getFailedShards() == 0) {
                                    nonCriticalExceptions.add("Count is " + sr.getHits().getTotalHits() + " but " + numDocs + " was expected. " + formatShardStatus(sr));
                                } else {
                                    assertHitCount(sr, numDocs);
                                }
                            }
                            final SearchHits sh = sr.getHits();
                            assertThat("Expected hits to be the same size the actual hits array", sh.getTotalHits(), equalTo((long) (sh.getHits().length)));
                        // this is the more critical but that we hit the actual hit array has a different size than the
                        // actual number of hits.
                        }
                    } catch (SearchPhaseExecutionException ex) {
                        // with replicas this should not happen
                        if (numberOfReplicas == 1 || !ex.getMessage().contains("all shards failed")) {
                            throw ex;
                        }
                    }
                }
            };
        }
        for (int j = 0; j < threads.length; j++) {
            threads[j].start();
        }
        allowNodes("test", between(1, 3));
        client().admin().cluster().prepareReroute().get();
        stop.set(true);
        for (int j = 0; j < threads.length; j++) {
            threads[j].join();
        }
        // this might time out on some machines if they are really busy and you hit lots of throttling
        ClusterHealthResponse resp = client().admin().cluster().prepareHealth().setWaitForYellowStatus().setWaitForNoRelocatingShards(true).setWaitForEvents(Priority.LANGUID).setTimeout("5m").get();
        assertNoTimeout(resp);
        // if we hit only non-critical exceptions we make sure that the post search works
        if (!nonCriticalExceptions.isEmpty()) {
            logger.info("non-critical exceptions: {}", nonCriticalExceptions);
            for (int j = 0; j < 10; j++) {
                assertHitCount(client().prepareSearch().get(), numDocs);
            }
        }
    }
}

Example 73 with CopyOnWriteArrayList

use of java.util.concurrent.CopyOnWriteArrayList in project elasticsearch by elastic.

the class ClusterServiceTests method testClusterStateUpdateTasksAreExecutedInOrder.

// test that for a single thread, tasks are executed in the order
// that they are submitted
public void testClusterStateUpdateTasksAreExecutedInOrder() throws BrokenBarrierException, InterruptedException {
    class TaskExecutor implements ClusterStateTaskExecutor<Integer> {

        List<Integer> tasks = new ArrayList<>();

        @Override
        public ClusterTasksResult<Integer> execute(ClusterState currentState, List<Integer> tasks) throws Exception {
            this.tasks.addAll(tasks);
            return ClusterTasksResult.<Integer>builder().successes(tasks).build(ClusterState.builder(currentState).build());
        }
    }
    int numberOfThreads = randomIntBetween(2, 8);
    TaskExecutor[] executors = new TaskExecutor[numberOfThreads];
    for (int i = 0; i < numberOfThreads; i++) {
        executors[i] = new TaskExecutor();
    }
    int tasksSubmittedPerThread = randomIntBetween(2, 1024);
    CopyOnWriteArrayList<Tuple<String, Throwable>> failures = new CopyOnWriteArrayList<>();
    CountDownLatch updateLatch = new CountDownLatch(numberOfThreads * tasksSubmittedPerThread);
    ClusterStateTaskListener listener = new ClusterStateTaskListener() {

        @Override
        public void onFailure(String source, Exception e) {
            logger.error((Supplier<?>) () -> new ParameterizedMessage("unexpected failure: [{}]", source), e);
            failures.add(new Tuple<>(source, e));
            updateLatch.countDown();
        }

        @Override
        public void clusterStateProcessed(String source, ClusterState oldState, ClusterState newState) {
            updateLatch.countDown();
        }
    };
    CyclicBarrier barrier = new CyclicBarrier(1 + numberOfThreads);
    for (int i = 0; i < numberOfThreads; i++) {
        final int index = i;
        Thread thread = new Thread(() -> {
            try {
                barrier.await();
                for (int j = 0; j < tasksSubmittedPerThread; j++) {
                    clusterService.submitStateUpdateTask("[" + index + "][" + j + "]", j, ClusterStateTaskConfig.build(randomFrom(Priority.values())), executors[index], listener);
                }
                barrier.await();
            } catch (InterruptedException | BrokenBarrierException e) {
                throw new AssertionError(e);
            }
        });
        thread.start();
    }
    // wait for all threads to be ready
    barrier.await();
    // wait for all threads to finish
    barrier.await();
    updateLatch.await();
    assertThat(failures, empty());
    for (int i = 0; i < numberOfThreads; i++) {
        assertEquals(tasksSubmittedPerThread, executors[i].tasks.size());
        for (int j = 0; j < tasksSubmittedPerThread; j++) {
            assertNotNull(executors[i].tasks.get(j));
            assertEquals("cluster state update task executed out of order", j, (int) executors[i].tasks.get(j));
        }
    }
}

Example 74 with CopyOnWriteArrayList

use of java.util.concurrent.CopyOnWriteArrayList in project elasticsearch by elastic.

the class UpdateIT method testStressUpdateDeleteConcurrency.

public void testStressUpdateDeleteConcurrency() throws Exception {
    //We create an index with merging disabled so that deletes don't get merged away
    assertAcked(prepareCreate("test").setSettings(Settings.builder().put(MergePolicyConfig.INDEX_MERGE_ENABLED, false)));
    ensureGreen();
    final int numberOfThreads = scaledRandomIntBetween(3, 5);
    final int numberOfIdsPerThread = scaledRandomIntBetween(3, 10);
    final int numberOfUpdatesPerId = scaledRandomIntBetween(10, 100);
    final int retryOnConflict = randomIntBetween(0, 1);
    final CountDownLatch latch = new CountDownLatch(numberOfThreads);
    final CountDownLatch startLatch = new CountDownLatch(1);
    final List<Throwable> failures = new CopyOnWriteArrayList<>();
    final class UpdateThread extends Thread {

        final Map<Integer, Integer> failedMap = new HashMap<>();

        final int numberOfIds;

        final int updatesPerId;

        final int maxUpdateRequests = numberOfIdsPerThread * numberOfUpdatesPerId;

        final int maxDeleteRequests = numberOfIdsPerThread * numberOfUpdatesPerId;

        private final Semaphore updateRequestsOutstanding = new Semaphore(maxUpdateRequests);

        private final Semaphore deleteRequestsOutstanding = new Semaphore(maxDeleteRequests);

        UpdateThread(int numberOfIds, int updatesPerId) {
            this.numberOfIds = numberOfIds;
            this.updatesPerId = updatesPerId;
        }

        final class UpdateListener implements ActionListener<UpdateResponse> {

            int id;

            UpdateListener(int id) {
                this.id = id;
            }

            @Override
            public void onResponse(UpdateResponse updateResponse) {
                updateRequestsOutstanding.release(1);
            }

            @Override
            public void onFailure(Exception e) {
                synchronized (failedMap) {
                    incrementMapValue(id, failedMap);
                }
                updateRequestsOutstanding.release(1);
            }
        }

        final class DeleteListener implements ActionListener<DeleteResponse> {

            int id;

            DeleteListener(int id) {
                this.id = id;
            }

            @Override
            public void onResponse(DeleteResponse deleteResponse) {
                deleteRequestsOutstanding.release(1);
            }

            @Override
            public void onFailure(Exception e) {
                synchronized (failedMap) {
                    incrementMapValue(id, failedMap);
                }
                deleteRequestsOutstanding.release(1);
            }
        }

        @Override
        public void run() {
            try {
                startLatch.await();
                boolean hasWaitedForNoNode = false;
                for (int j = 0; j < numberOfIds; j++) {
                    for (int k = 0; k < numberOfUpdatesPerId; ++k) {
                        updateRequestsOutstanding.acquire();
                        try {
                            UpdateRequest ur = client().prepareUpdate("test", "type1", Integer.toString(j)).setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).setRetryOnConflict(retryOnConflict).setUpsert(jsonBuilder().startObject().field("field", 1).endObject()).request();
                            client().update(ur, new UpdateListener(j));
                        } catch (NoNodeAvailableException nne) {
                            updateRequestsOutstanding.release();
                            synchronized (failedMap) {
                                incrementMapValue(j, failedMap);
                            }
                            if (hasWaitedForNoNode) {
                                throw nne;
                            }
                            logger.warn("Got NoNodeException waiting for 1 second for things to recover.");
                            hasWaitedForNoNode = true;
                            Thread.sleep(1000);
                        }
                        try {
                            deleteRequestsOutstanding.acquire();
                            DeleteRequest dr = client().prepareDelete("test", "type1", Integer.toString(j)).request();
                            client().delete(dr, new DeleteListener(j));
                        } catch (NoNodeAvailableException nne) {
                            deleteRequestsOutstanding.release();
                            synchronized (failedMap) {
                                incrementMapValue(j, failedMap);
                            }
                            if (hasWaitedForNoNode) {
                                throw nne;
                            }
                            logger.warn("Got NoNodeException waiting for 1 second for things to recover.");
                            hasWaitedForNoNode = true;
                            //Wait for no-node to clear
                            Thread.sleep(1000);
                        }
                    }
                }
            } catch (Exception e) {
                logger.error("Something went wrong", e);
                failures.add(e);
            } finally {
                try {
                    waitForOutstandingRequests(TimeValue.timeValueSeconds(60), updateRequestsOutstanding, maxUpdateRequests, "Update");
                    waitForOutstandingRequests(TimeValue.timeValueSeconds(60), deleteRequestsOutstanding, maxDeleteRequests, "Delete");
                } catch (ElasticsearchTimeoutException ete) {
                    failures.add(ete);
                }
                latch.countDown();
            }
        }

        private void incrementMapValue(int j, Map<Integer, Integer> map) {
            if (!map.containsKey(j)) {
                map.put(j, 0);
            }
            map.put(j, map.get(j) + 1);
        }

        private void waitForOutstandingRequests(TimeValue timeOut, Semaphore requestsOutstanding, int maxRequests, String name) {
            long start = System.currentTimeMillis();
            do {
                long msRemaining = timeOut.getMillis() - (System.currentTimeMillis() - start);
                logger.info("[{}] going to try and acquire [{}] in [{}]ms [{}] available to acquire right now", name, maxRequests, msRemaining, requestsOutstanding.availablePermits());
                try {
                    requestsOutstanding.tryAcquire(maxRequests, msRemaining, TimeUnit.MILLISECONDS);
                    return;
                } catch (InterruptedException ie) {
                //Just keep swimming
                }
            } while ((System.currentTimeMillis() - start) < timeOut.getMillis());
            throw new ElasticsearchTimeoutException("Requests were still outstanding after the timeout [" + timeOut + "] for type [" + name + "]");
        }
    }
    final List<UpdateThread> threads = new ArrayList<>();
    for (int i = 0; i < numberOfThreads; i++) {
        UpdateThread ut = new UpdateThread(numberOfIdsPerThread, numberOfUpdatesPerId);
        ut.start();
        threads.add(ut);
    }
    startLatch.countDown();
    latch.await();
    for (UpdateThread ut : threads) {
        //Threads should have finished because of the latch.await
        ut.join();
    }
    //aquiring the request outstanding semaphores.
    for (Throwable throwable : failures) {
        logger.info("Captured failure on concurrent update:", throwable);
    }
    assertThat(failures.size(), equalTo(0));
    //All the previous operations should be complete or failed at this point
    for (int i = 0; i < numberOfIdsPerThread; ++i) {
        UpdateResponse ur = client().prepareUpdate("test", "type1", Integer.toString(i)).setScript(new Script(ScriptType.INLINE, "field_inc", "field", Collections.emptyMap())).setRetryOnConflict(Integer.MAX_VALUE).setUpsert(jsonBuilder().startObject().field("field", 1).endObject()).execute().actionGet();
    }
    refresh();
    for (int i = 0; i < numberOfIdsPerThread; ++i) {
        int totalFailures = 0;
        GetResponse response = client().prepareGet("test", "type1", Integer.toString(i)).execute().actionGet();
        if (response.isExists()) {
            assertThat(response.getId(), equalTo(Integer.toString(i)));
            int expectedVersion = (numberOfThreads * numberOfUpdatesPerId * 2) + 1;
            for (UpdateThread ut : threads) {
                if (ut.failedMap.containsKey(i)) {
                    totalFailures += ut.failedMap.get(i);
                }
            }
            expectedVersion -= totalFailures;
            logger.error("Actual version [{}] Expected version [{}] Total failures [{}]", response.getVersion(), expectedVersion, totalFailures);
            assertThat(response.getVersion(), equalTo((long) expectedVersion));
            assertThat(response.getVersion() + totalFailures, equalTo((long) ((numberOfUpdatesPerId * numberOfThreads * 2) + 1)));
        }
    }
}

Example 75 with CopyOnWriteArrayList

use of java.util.concurrent.CopyOnWriteArrayList in project hazelcast by hazelcast.

the class DiscoverySpiTest method getDiscoverySPIConfig.

private static Config getDiscoverySPIConfig(String xmlFileName) {
    InputStream xmlResource = DiscoverySpiTest.class.getClassLoader().getResourceAsStream(xmlFileName);
    Config config = new XmlConfigBuilder(xmlResource).build();
    config.getNetworkConfig().setPort(50001);
    InterfacesConfig interfaces = config.getNetworkConfig().getInterfaces();
    interfaces.clear();
    interfaces.setEnabled(true);
    interfaces.addInterface("127.0.0.1");
    List<DiscoveryNode> discoveryNodes = new CopyOnWriteArrayList<DiscoveryNode>();
    DiscoveryStrategyFactory factory = new CollectingDiscoveryStrategyFactory(discoveryNodes);
    DiscoveryConfig discoveryConfig = config.getNetworkConfig().getJoin().getDiscoveryConfig();
    discoveryConfig.getDiscoveryStrategyConfigs().clear();
    DiscoveryStrategyConfig strategyConfig = new DiscoveryStrategyConfig(factory, Collections.<String, Comparable>emptyMap());
    discoveryConfig.addDiscoveryStrategyConfig(strategyConfig);
    return config;
}