Examples with BlockCache org.apache.solr.store.blockcache.BlockCache used on opensource projects

Example 1 with BlockCache

use of org.apache.solr.store.blockcache.BlockCache in project lucene-solr by apache.

the class HdfsWriteToMultipleCollectionsTest method test.

@Test
public void test() throws Exception {
    int docCount = random().nextInt(1313) + 1;
    int cnt = random().nextInt(4) + 1;
    for (int i = 0; i < cnt; i++) {
        createCollection(ACOLLECTION + i, 2, 2, 9);
    }
    for (int i = 0; i < cnt; i++) {
        waitForRecoveriesToFinish(ACOLLECTION + i, false);
    }
    List<CloudSolrClient> cloudClients = new ArrayList<>();
    List<StoppableIndexingThread> threads = new ArrayList<>();
    for (int i = 0; i < cnt; i++) {
        CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress());
        client.setDefaultCollection(ACOLLECTION + i);
        cloudClients.add(client);
        StoppableIndexingThread indexThread = new StoppableIndexingThread(null, client, "1", true, docCount, 1, true);
        threads.add(indexThread);
        indexThread.start();
    }
    int addCnt = 0;
    for (StoppableIndexingThread thread : threads) {
        thread.join();
        addCnt += thread.getNumAdds() - thread.getNumDeletes();
    }
    long collectionsCount = 0;
    for (CloudSolrClient client : cloudClients) {
        client.commit();
        collectionsCount += client.query(new SolrQuery("*:*")).getResults().getNumFound();
    }
    IOUtils.close(cloudClients);
    assertEquals(addCnt, collectionsCount);
    BlockCache lastBlockCache = null;
    // assert that we are using the block directory and that write and read caching are being used
    for (JettySolrRunner jetty : jettys) {
        CoreContainer cores = jetty.getCoreContainer();
        Collection<SolrCore> solrCores = cores.getCores();
        for (SolrCore core : solrCores) {
            if (core.getCoreDescriptor().getCloudDescriptor().getCollectionName().startsWith(ACOLLECTION)) {
                DirectoryFactory factory = core.getDirectoryFactory();
                assertTrue("Found: " + core.getDirectoryFactory().getClass().getName(), factory instanceof HdfsDirectoryFactory);
                Directory dir = factory.get(core.getDataDir(), null, null);
                try {
                    long dataDirSize = factory.size(dir);
                    FileSystem fileSystem = null;
                    fileSystem = FileSystem.newInstance(new Path(core.getDataDir()).toUri(), new Configuration());
                    long size = fileSystem.getContentSummary(new Path(core.getDataDir())).getLength();
                    assertEquals(size, dataDirSize);
                } finally {
                    core.getDirectoryFactory().release(dir);
                }
                RefCounted<IndexWriter> iwRef = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core);
                try {
                    IndexWriter iw = iwRef.get();
                    NRTCachingDirectory directory = (NRTCachingDirectory) iw.getDirectory();
                    BlockDirectory blockDirectory = (BlockDirectory) directory.getDelegate();
                    assertTrue(blockDirectory.isBlockCacheReadEnabled());
                    // see SOLR-6424
                    assertFalse(blockDirectory.isBlockCacheWriteEnabled());
                    Cache cache = blockDirectory.getCache();
                    // we know it's a BlockDirectoryCache, but future proof
                    assertTrue(cache instanceof BlockDirectoryCache);
                    BlockCache blockCache = ((BlockDirectoryCache) cache).getBlockCache();
                    if (lastBlockCache != null) {
                        if (Boolean.getBoolean(SOLR_HDFS_BLOCKCACHE_GLOBAL)) {
                            assertEquals(lastBlockCache, blockCache);
                        } else {
                            assertNotSame(lastBlockCache, blockCache);
                        }
                    }
                    lastBlockCache = blockCache;
                } finally {
                    iwRef.decref();
                }
            }
        }
    }
}

Example 2 with BlockCache

use of org.apache.solr.store.blockcache.BlockCache in project lucene-solr by apache.

the class HdfsDirectoryFactory method createBlockCache.

private BlockCache createBlockCache(int numberOfBlocksPerBank, int blockSize, int bankCount, boolean directAllocation, int slabSize, int bufferSize, int bufferCount) {
    BufferStore.initNewBuffer(bufferSize, bufferCount, metrics);
    long totalMemory = (long) bankCount * (long) numberOfBlocksPerBank * (long) blockSize;
    BlockCache blockCache;
    try {
        blockCache = new BlockCache(metrics, directAllocation, totalMemory, slabSize, blockSize);
    } catch (OutOfMemoryError e) {
        throw new RuntimeException("The max direct memory is likely too low.  Either increase it (by adding -XX:MaxDirectMemorySize=<size>g -XX:+UseLargePages to your containers startup args)" + " or disable direct allocation using solr.hdfs.blockcache.direct.memory.allocation=false in solrconfig.xml. If you are putting the block cache on the heap," + " your java heap size might not be large enough." + " Failed allocating ~" + totalMemory / 1000000.0 + " MB.", e);
    }
    return blockCache;
}

Example 3 with BlockCache

use of org.apache.solr.store.blockcache.BlockCache in project lucene-solr by apache.

the class HdfsDirectoryFactory method create.

@Override
protected Directory create(String path, LockFactory lockFactory, DirContext dirContext) throws IOException {
    assert params != null : "init must be called before create";
    LOG.info("creating directory factory for path {}", path);
    Configuration conf = getConf();
    if (metrics == null) {
        metrics = MetricsHolder.metrics;
    }
    boolean blockCacheEnabled = getConfig(BLOCKCACHE_ENABLED, true);
    boolean blockCacheGlobal = getConfig(BLOCKCACHE_GLOBAL, true);
    boolean blockCacheReadEnabled = getConfig(BLOCKCACHE_READ_ENABLED, true);
    final HdfsDirectory hdfsDir;
    final Directory dir;
    if (blockCacheEnabled && dirContext != DirContext.META_DATA) {
        int numberOfBlocksPerBank = getConfig(NUMBEROFBLOCKSPERBANK, 16384);
        int blockSize = BlockDirectory.BLOCK_SIZE;
        int bankCount = getConfig(BLOCKCACHE_SLAB_COUNT, 1);
        boolean directAllocation = getConfig(BLOCKCACHE_DIRECT_MEMORY_ALLOCATION, true);
        int slabSize = numberOfBlocksPerBank * blockSize;
        LOG.info("Number of slabs of block cache [{}] with direct memory allocation set to [{}]", bankCount, directAllocation);
        LOG.info("Block cache target memory usage, slab size of [{}] will allocate [{}] slabs and use ~[{}] bytes", new Object[] { slabSize, bankCount, ((long) bankCount * (long) slabSize) });
        int bsBufferSize = params.getInt("solr.hdfs.blockcache.bufferstore.buffersize", blockSize);
        // this is actually total size
        int bsBufferCount = params.getInt("solr.hdfs.blockcache.bufferstore.buffercount", 0);
        BlockCache blockCache = getBlockDirectoryCache(numberOfBlocksPerBank, blockSize, bankCount, directAllocation, slabSize, bsBufferSize, bsBufferCount, blockCacheGlobal);
        Cache cache = new BlockDirectoryCache(blockCache, path, metrics, blockCacheGlobal);
        int readBufferSize = params.getInt("solr.hdfs.blockcache.read.buffersize", blockSize);
        hdfsDir = new HdfsDirectory(new Path(path), lockFactory, conf, readBufferSize);
        dir = new BlockDirectory(path, hdfsDir, cache, null, blockCacheReadEnabled, false, cacheMerges, cacheReadOnce);
    } else {
        hdfsDir = new HdfsDirectory(new Path(path), conf);
        dir = hdfsDir;
    }
    if (params.getBool(LOCALITYMETRICS_ENABLED, false)) {
        LocalityHolder.reporter.registerDirectory(hdfsDir);
    }
    boolean nrtCachingDirectory = getConfig(NRTCACHINGDIRECTORY_ENABLE, true);
    if (nrtCachingDirectory) {
        double nrtCacheMaxMergeSizeMB = getConfig(NRTCACHINGDIRECTORY_MAXMERGESIZEMB, 16);
        double nrtCacheMaxCacheMB = getConfig(NRTCACHINGDIRECTORY_MAXCACHEMB, 192);
        return new NRTCachingDirectory(dir, nrtCacheMaxMergeSizeMB, nrtCacheMaxCacheMB);
    }
    return dir;
}