Examples with DefaultSolrThreadFactory org.apache.solr.util.DefaultSolrThreadFactory used on opensource projects

Example 6 with DefaultSolrThreadFactory

use of org.apache.solr.util.DefaultSolrThreadFactory in project lucene-solr by apache.

the class TestStressLiveNodes method testStress.

public void testStress() throws Exception {
    // do many iters, so we have "bursts" of adding nodes that we then check
    final int numIters = atLeast(TEST_NIGHTLY ? 1000 : 100);
    for (int iter = 0; iter < numIters; iter++) {
        // sanity check that ZK says there is in fact 1 live node
        List<String> actualLiveNodes = getTrueLiveNodesFromZk();
        assertEquals("iter" + iter + ": " + actualLiveNodes.toString(), 1, actualLiveNodes.size());
        // only here do we forcibly update the cached live nodes so we don't have to wait for it to catch up
        // with all the ephemeral nodes that vanished after the last iteration
        CLOUD_CLIENT.getZkStateReader().updateLiveNodes();
        // sanity check that our Cloud Client's local state knows about the 1 (real) live node in our cluster
        List<String> cachedLiveNodes = getCachedLiveNodesFromLocalState(actualLiveNodes.size());
        assertEquals("iter" + iter + " " + actualLiveNodes.size() + " != " + cachedLiveNodes.size(), actualLiveNodes, cachedLiveNodes);
        // start spinning up some threads to add some live_node children in parallel
        // we don't need a lot of threads or nodes (we don't want to swamp the CPUs
        // just bursts of concurrent adds) but we do want to randomize it a bit so we increase the
        // odds of concurrent watchers firing regardless of the num CPUs or load on the machine running
        // the test (but we deliberately don't look at availableProcessors() since we want randomization
        // consistency across all machines for a given seed)
        final int numThreads = TestUtil.nextInt(random(), 2, 5);
        // use same num for all thrashers, to increase likely hood of them all competing
        // (diff random number would mean heavy concurrency only for ~ the first N=lowest num requests)
        //
        // this does not need to be a large number -- in fact, the higher it is, the more
        // likely we are to see a mistake in early watcher triggers get "corrected" by a later one
        // and overlook a possible bug
        final int numNodesPerThrasher = TestUtil.nextInt(random(), 1, 5);
        log.info("preparing parallel adds to live nodes: iter={}, numThreads={} numNodesPerThread={}", iter, numThreads, numNodesPerThrasher);
        // NOTE: using ephemeral nodes
        // so we can't close any of these thrashers until we are done with our assertions
        final List<LiveNodeTrasher> thrashers = new ArrayList<>(numThreads);
        for (int i = 0; i < numThreads; i++) {
            thrashers.add(new LiveNodeTrasher("T" + iter + "_" + i, numNodesPerThrasher));
        }
        try {
            final ExecutorService executorService = ExecutorUtil.newMDCAwareFixedThreadPool(thrashers.size() + 1, new DefaultSolrThreadFactory("test_live_nodes_thrasher_iter" + iter));
            executorService.invokeAll(thrashers);
            executorService.shutdown();
            if (!executorService.awaitTermination(WAIT_TIME, TimeUnit.SECONDS)) {
                for (LiveNodeTrasher thrasher : thrashers) {
                    thrasher.stop();
                }
            }
            assertTrue("iter" + iter + ": thrashers didn't finish even after explicitly stopping", executorService.awaitTermination(WAIT_TIME, TimeUnit.SECONDS));
            // sanity check the *real* live_nodes entries from ZK match what the thrashers added
            // 1 real live node when we started
            int totalAdded = 1;
            for (LiveNodeTrasher thrasher : thrashers) {
                totalAdded += thrasher.getNumAdded();
            }
            actualLiveNodes = getTrueLiveNodesFromZk();
            assertEquals("iter" + iter, totalAdded, actualLiveNodes.size());
            // verify our local client knows the correct set of live nodes
            cachedLiveNodes = getCachedLiveNodesFromLocalState(actualLiveNodes.size());
            assertEquals("iter" + iter + " " + actualLiveNodes.size() + " != " + cachedLiveNodes.size(), actualLiveNodes, cachedLiveNodes);
        } finally {
            for (LiveNodeTrasher thrasher : thrashers) {
                // shutdown our zk connection, freeing our ephemeral nodes
                thrasher.close();
            }
        }
    }
}

Example 7 with DefaultSolrThreadFactory

use of org.apache.solr.util.DefaultSolrThreadFactory in project lucene-solr by apache.

the class TestLFUCache method testConcurrentAccess.

@Test
public void testConcurrentAccess() throws InterruptedException {
    /* Set up a thread pool with twice as many threads as there are CPUs. */
    final ConcurrentLFUCache<Integer, Long> cache = new ConcurrentLFUCache<>(10, 9);
    ExecutorService executorService = ExecutorUtil.newMDCAwareFixedThreadPool(10, new DefaultSolrThreadFactory("testConcurrentAccess"));
    final AtomicReference<Throwable> error = new AtomicReference<>();
    /*
     * Use the thread pool to execute at least two million puts into the cache.
     * Without the fix on SOLR-7585, NoSuchElementException is thrown.
     * Simultaneous calls to markAndSweep are protected from each other by a
     * lock, so they run sequentially, and due to a problem in the previous
     * design, the cache eviction doesn't work right.
     */
    for (int i = 0; i < atLeast(2_000_000); ++i) {
        executorService.submit(() -> {
            try {
                cache.put(random().nextInt(100), random().nextLong());
            } catch (Throwable t) {
                error.compareAndSet(null, t);
            }
        });
    }
    executorService.shutdown();
    executorService.awaitTermination(1, TimeUnit.MINUTES);
    // then:
    assertNull("Exception during concurrent access: " + error.get(), error.get());
}

Example 8 with DefaultSolrThreadFactory

use of org.apache.solr.util.DefaultSolrThreadFactory in project lucene-solr by apache.

the class AddBlockUpdateTest method beforeClass.

@BeforeClass
public static void beforeClass() throws Exception {
    String oldCacheNamePropValue = System.getProperty("blockJoinParentFilterCache");
    System.setProperty("blockJoinParentFilterCache", (cachedMode = random().nextBoolean()) ? "blockJoinParentFilterCache" : "don't cache");
    if (oldCacheNamePropValue != null) {
        System.setProperty("blockJoinParentFilterCache", oldCacheNamePropValue);
    }
    inputFactory = XMLInputFactory.newInstance();
    // Executors.newSingleThreadExecutor();
    exe = rarely() ? ExecutorUtil.newMDCAwareFixedThreadPool(atLeast(2), new DefaultSolrThreadFactory("AddBlockUpdateTest")) : ExecutorUtil.newMDCAwareCachedThreadPool(new DefaultSolrThreadFactory("AddBlockUpdateTest"));
    initCore("solrconfig.xml", "schema15.xml");
}

Example 9 with DefaultSolrThreadFactory

use of org.apache.solr.util.DefaultSolrThreadFactory in project lucene-solr by apache.

the class OverseerTest method testShardAssignmentBigger.

@Test
public void testShardAssignmentBigger() throws Exception {
    String zkDir = createTempDir("zkData").toFile().getAbsolutePath();
    //how many simulated nodes (num of threads)
    final int nodeCount = random().nextInt(TEST_NIGHTLY ? 50 : 10) + (TEST_NIGHTLY ? 50 : 10) + 1;
    //how many cores to register
    final int coreCount = random().nextInt(TEST_NIGHTLY ? 100 : 11) + (TEST_NIGHTLY ? 100 : 11) + 1;
    //how many slices
    final int sliceCount = random().nextInt(TEST_NIGHTLY ? 20 : 5) + 1;
    ZkTestServer server = new ZkTestServer(zkDir);
    SolrZkClient zkClient = null;
    ZkStateReader reader = null;
    SolrZkClient overseerClient = null;
    final MockZKController[] controllers = new MockZKController[nodeCount];
    final ExecutorService[] nodeExecutors = new ExecutorService[nodeCount];
    try {
        server.run();
        AbstractZkTestCase.tryCleanSolrZkNode(server.getZkHost());
        AbstractZkTestCase.makeSolrZkNode(server.getZkHost());
        zkClient = new SolrZkClient(server.getZkAddress(), TIMEOUT);
        ZkController.createClusterZkNodes(zkClient);
        overseerClient = electNewOverseer(server.getZkAddress());
        reader = new ZkStateReader(zkClient);
        reader.createClusterStateWatchersAndUpdate();
        for (int i = 0; i < nodeCount; i++) {
            controllers[i] = new MockZKController(server.getZkAddress(), "node" + i);
        }
        for (int i = 0; i < nodeCount; i++) {
            nodeExecutors[i] = ExecutorUtil.newMDCAwareFixedThreadPool(1, new DefaultSolrThreadFactory("testShardAssignment"));
        }
        final String[] ids = new String[coreCount];
        //register total of coreCount cores
        for (int i = 0; i < coreCount; i++) {
            final int slot = i;
            nodeExecutors[i % nodeCount].submit((Runnable) () -> {
                final String coreName = "core" + slot;
                try {
                    ids[slot] = controllers[slot % nodeCount].publishState(COLLECTION, coreName, "node" + slot, Replica.State.ACTIVE, sliceCount);
                } catch (Throwable e) {
                    e.printStackTrace();
                    fail("register threw exception:" + e.getClass());
                }
            });
        }
        for (int i = 0; i < nodeCount; i++) {
            nodeExecutors[i].shutdown();
        }
        for (int i = 0; i < nodeCount; i++) {
            while (!nodeExecutors[i].awaitTermination(100, TimeUnit.MILLISECONDS)) ;
        }
        // make sure all cores have been assigned a id in cloudstate
        int cloudStateSliceCount = 0;
        for (int i = 0; i < 40; i++) {
            cloudStateSliceCount = 0;
            ClusterState state = reader.getClusterState();
            final Map<String, Slice> slices = state.getSlicesMap(COLLECTION);
            if (slices != null) {
                for (String name : slices.keySet()) {
                    cloudStateSliceCount += slices.get(name).getReplicasMap().size();
                }
                if (coreCount == cloudStateSliceCount)
                    break;
            }
            Thread.sleep(200);
        }
        assertEquals("Unable to verify all cores have been assigned an id in cloudstate", coreCount, cloudStateSliceCount);
        // make sure all cores have been returned an id
        int assignedCount = 0;
        for (int i = 0; i < 240; i++) {
            assignedCount = 0;
            for (int j = 0; j < coreCount; j++) {
                if (ids[j] != null) {
                    assignedCount++;
                }
            }
            if (coreCount == assignedCount) {
                break;
            }
            Thread.sleep(1000);
        }
        assertEquals("Unable to verify all cores have been returned an id", coreCount, assignedCount);
        final HashMap<String, AtomicInteger> counters = new HashMap<>();
        for (int i = 1; i < sliceCount + 1; i++) {
            counters.put("shard" + i, new AtomicInteger());
        }
        for (int i = 0; i < coreCount; i++) {
            final AtomicInteger ai = counters.get(ids[i]);
            assertNotNull("could not find counter for shard:" + ids[i], ai);
            ai.incrementAndGet();
        }
        for (String counter : counters.keySet()) {
            int count = counters.get(counter).intValue();
            int expectedCount = coreCount / sliceCount;
            int min = expectedCount - 1;
            int max = expectedCount + 1;
            if (count < min || count > max) {
                fail("Unevenly assigned shard ids, " + counter + " had " + count + ", expected: " + min + "-" + max);
            }
        }
        //make sure leaders are in cloud state
        for (int i = 0; i < sliceCount; i++) {
            assertNotNull(reader.getLeaderUrl(COLLECTION, "shard" + (i + 1), 15000));
        }
    } finally {
        close(zkClient);
        close(overseerClient);
        close(reader);
        for (int i = 0; i < controllers.length; i++) if (controllers[i] != null) {
            controllers[i].close();
        }
        server.shutdown();
        for (int i = 0; i < nodeCount; i++) {
            if (nodeExecutors[i] != null) {
                nodeExecutors[i].shutdownNow();
            }
        }
    }
}

Example 10 with DefaultSolrThreadFactory

use of org.apache.solr.util.DefaultSolrThreadFactory in project lucene-solr by apache.

the class UnloadDistributedZkTest method testUnloadLotsOfCores.

private void testUnloadLotsOfCores() throws Exception {
    SolrClient client = clients.get(2);
    String url3 = getBaseUrl(client);
    try (final HttpSolrClient adminClient = getHttpSolrClient(url3)) {
        adminClient.setConnectionTimeout(15000);
        adminClient.setSoTimeout(60000);
        int cnt = atLeast(3);
        ThreadPoolExecutor executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0, Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new DefaultSolrThreadFactory("testExecutor"));
        try {
            // create the cores
            createCores(adminClient, executor, "multiunload", 2, cnt);
        } finally {
            ExecutorUtil.shutdownAndAwaitTermination(executor);
        }
        executor = new ExecutorUtil.MDCAwareThreadPoolExecutor(0, Integer.MAX_VALUE, 5, TimeUnit.SECONDS, new SynchronousQueue<Runnable>(), new DefaultSolrThreadFactory("testExecutor"));
        try {
            for (int j = 0; j < cnt; j++) {
                final int freezeJ = j;
                executor.execute(() -> {
                    Unload unloadCmd = new Unload(true);
                    unloadCmd.setCoreName("multiunload" + freezeJ);
                    try {
                        adminClient.request(unloadCmd);
                    } catch (SolrServerException | IOException e) {
                        throw new RuntimeException(e);
                    }
                });
                Thread.sleep(random().nextInt(50));
            }
        } finally {
            ExecutorUtil.shutdownAndAwaitTermination(executor);
        }
    }
}