Search in sources :

Example 1 with StoppableIndexingThread

use of org.apache.solr.cloud.StoppableIndexingThread in project lucene-solr by apache.

the class HdfsWriteToMultipleCollectionsTest method test.

@Test
public void test() throws Exception {
    int docCount = random().nextInt(1313) + 1;
    int cnt = random().nextInt(4) + 1;
    for (int i = 0; i < cnt; i++) {
        createCollection(ACOLLECTION + i, 2, 2, 9);
    }
    for (int i = 0; i < cnt; i++) {
        waitForRecoveriesToFinish(ACOLLECTION + i, false);
    }
    List<CloudSolrClient> cloudClients = new ArrayList<>();
    List<StoppableIndexingThread> threads = new ArrayList<>();
    for (int i = 0; i < cnt; i++) {
        CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress());
        client.setDefaultCollection(ACOLLECTION + i);
        cloudClients.add(client);
        StoppableIndexingThread indexThread = new StoppableIndexingThread(null, client, "1", true, docCount, 1, true);
        threads.add(indexThread);
        indexThread.start();
    }
    int addCnt = 0;
    for (StoppableIndexingThread thread : threads) {
        thread.join();
        addCnt += thread.getNumAdds() - thread.getNumDeletes();
    }
    long collectionsCount = 0;
    for (CloudSolrClient client : cloudClients) {
        client.commit();
        collectionsCount += client.query(new SolrQuery("*:*")).getResults().getNumFound();
    }
    IOUtils.close(cloudClients);
    assertEquals(addCnt, collectionsCount);
    BlockCache lastBlockCache = null;
    // assert that we are using the block directory and that write and read caching are being used
    for (JettySolrRunner jetty : jettys) {
        CoreContainer cores = jetty.getCoreContainer();
        Collection<SolrCore> solrCores = cores.getCores();
        for (SolrCore core : solrCores) {
            if (core.getCoreDescriptor().getCloudDescriptor().getCollectionName().startsWith(ACOLLECTION)) {
                DirectoryFactory factory = core.getDirectoryFactory();
                assertTrue("Found: " + core.getDirectoryFactory().getClass().getName(), factory instanceof HdfsDirectoryFactory);
                Directory dir = factory.get(core.getDataDir(), null, null);
                try {
                    long dataDirSize = factory.size(dir);
                    FileSystem fileSystem = null;
                    fileSystem = FileSystem.newInstance(new Path(core.getDataDir()).toUri(), new Configuration());
                    long size = fileSystem.getContentSummary(new Path(core.getDataDir())).getLength();
                    assertEquals(size, dataDirSize);
                } finally {
                    core.getDirectoryFactory().release(dir);
                }
                RefCounted<IndexWriter> iwRef = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core);
                try {
                    IndexWriter iw = iwRef.get();
                    NRTCachingDirectory directory = (NRTCachingDirectory) iw.getDirectory();
                    BlockDirectory blockDirectory = (BlockDirectory) directory.getDelegate();
                    assertTrue(blockDirectory.isBlockCacheReadEnabled());
                    // see SOLR-6424
                    assertFalse(blockDirectory.isBlockCacheWriteEnabled());
                    Cache cache = blockDirectory.getCache();
                    // we know it's a BlockDirectoryCache, but future proof
                    assertTrue(cache instanceof BlockDirectoryCache);
                    BlockCache blockCache = ((BlockDirectoryCache) cache).getBlockCache();
                    if (lastBlockCache != null) {
                        if (Boolean.getBoolean(SOLR_HDFS_BLOCKCACHE_GLOBAL)) {
                            assertEquals(lastBlockCache, blockCache);
                        } else {
                            assertNotSame(lastBlockCache, blockCache);
                        }
                    }
                    lastBlockCache = blockCache;
                } finally {
                    iwRef.decref();
                }
            }
        }
    }
}
Also used : Configuration(org.apache.hadoop.conf.Configuration) SolrCore(org.apache.solr.core.SolrCore) BlockDirectoryCache(org.apache.solr.store.blockcache.BlockDirectoryCache) ArrayList(java.util.ArrayList) HdfsDirectoryFactory(org.apache.solr.core.HdfsDirectoryFactory) SolrQuery(org.apache.solr.client.solrj.SolrQuery) CloudSolrClient(org.apache.solr.client.solrj.impl.CloudSolrClient) StoppableIndexingThread(org.apache.solr.cloud.StoppableIndexingThread) CoreContainer(org.apache.solr.core.CoreContainer) DirectoryFactory(org.apache.solr.core.DirectoryFactory) HdfsDirectoryFactory(org.apache.solr.core.HdfsDirectoryFactory) FileSystem(org.apache.hadoop.fs.FileSystem) BlockDirectory(org.apache.solr.store.blockcache.BlockDirectory) Directory(org.apache.lucene.store.Directory) NRTCachingDirectory(org.apache.lucene.store.NRTCachingDirectory) Path(org.apache.hadoop.fs.Path) JettySolrRunner(org.apache.solr.client.solrj.embedded.JettySolrRunner) NRTCachingDirectory(org.apache.lucene.store.NRTCachingDirectory) BlockDirectory(org.apache.solr.store.blockcache.BlockDirectory) IndexWriter(org.apache.lucene.index.IndexWriter) BlockCache(org.apache.solr.store.blockcache.BlockCache) Cache(org.apache.solr.store.blockcache.Cache) BlockDirectoryCache(org.apache.solr.store.blockcache.BlockDirectoryCache) BlockCache(org.apache.solr.store.blockcache.BlockCache) BasicDistributedZkTest(org.apache.solr.cloud.BasicDistributedZkTest) Test(org.junit.Test)

Aggregations

ArrayList (java.util.ArrayList)1 Configuration (org.apache.hadoop.conf.Configuration)1 FileSystem (org.apache.hadoop.fs.FileSystem)1 Path (org.apache.hadoop.fs.Path)1 IndexWriter (org.apache.lucene.index.IndexWriter)1 Directory (org.apache.lucene.store.Directory)1 NRTCachingDirectory (org.apache.lucene.store.NRTCachingDirectory)1 SolrQuery (org.apache.solr.client.solrj.SolrQuery)1 JettySolrRunner (org.apache.solr.client.solrj.embedded.JettySolrRunner)1 CloudSolrClient (org.apache.solr.client.solrj.impl.CloudSolrClient)1 BasicDistributedZkTest (org.apache.solr.cloud.BasicDistributedZkTest)1 StoppableIndexingThread (org.apache.solr.cloud.StoppableIndexingThread)1 CoreContainer (org.apache.solr.core.CoreContainer)1 DirectoryFactory (org.apache.solr.core.DirectoryFactory)1 HdfsDirectoryFactory (org.apache.solr.core.HdfsDirectoryFactory)1 SolrCore (org.apache.solr.core.SolrCore)1 BlockCache (org.apache.solr.store.blockcache.BlockCache)1 BlockDirectory (org.apache.solr.store.blockcache.BlockDirectory)1 BlockDirectoryCache (org.apache.solr.store.blockcache.BlockDirectoryCache)1 Cache (org.apache.solr.store.blockcache.Cache)1