Examples with DirectoryFactory org.apache.solr.core.DirectoryFactory used on opensource projects

Example 1 with DirectoryFactory

use of org.apache.solr.core.DirectoryFactory in project lucene-solr by apache.

the class MergeIndexesOp method execute.

@Override
public void execute(CoreAdminHandler.CallInfo it) throws Exception {
    SolrParams params = it.req.getParams();
    String cname = params.required().get(CoreAdminParams.CORE);
    SolrCore core = it.handler.coreContainer.getCore(cname);
    SolrQueryRequest wrappedReq = null;
    List<SolrCore> sourceCores = Lists.newArrayList();
    List<RefCounted<SolrIndexSearcher>> searchers = Lists.newArrayList();
    // stores readers created from indexDir param values
    List<DirectoryReader> readersToBeClosed = Lists.newArrayList();
    Map<Directory, Boolean> dirsToBeReleased = new HashMap<>();
    if (core != null) {
        try {
            String[] dirNames = params.getParams(CoreAdminParams.INDEX_DIR);
            if (dirNames == null || dirNames.length == 0) {
                String[] sources = params.getParams("srcCore");
                if (sources == null || sources.length == 0)
                    throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "At least one indexDir or srcCore must be specified");
                for (int i = 0; i < sources.length; i++) {
                    String source = sources[i];
                    SolrCore srcCore = it.handler.coreContainer.getCore(source);
                    if (srcCore == null)
                        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Core: " + source + " does not exist");
                    sourceCores.add(srcCore);
                }
            } else {
                DirectoryFactory dirFactory = core.getDirectoryFactory();
                for (int i = 0; i < dirNames.length; i++) {
                    boolean markAsDone = false;
                    if (dirFactory instanceof CachingDirectoryFactory) {
                        if (!((CachingDirectoryFactory) dirFactory).getLivePaths().contains(dirNames[i])) {
                            markAsDone = true;
                        }
                    }
                    Directory dir = dirFactory.get(dirNames[i], DirectoryFactory.DirContext.DEFAULT, core.getSolrConfig().indexConfig.lockType);
                    dirsToBeReleased.put(dir, markAsDone);
                    // TODO: why doesn't this use the IR factory? what is going on here?
                    readersToBeClosed.add(DirectoryReader.open(dir));
                }
            }
            List<DirectoryReader> readers = null;
            if (readersToBeClosed.size() > 0) {
                readers = readersToBeClosed;
            } else {
                readers = Lists.newArrayList();
                for (SolrCore solrCore : sourceCores) {
                    // record the searchers so that we can decref
                    RefCounted<SolrIndexSearcher> searcher = solrCore.getSearcher();
                    searchers.add(searcher);
                    readers.add(searcher.get().getIndexReader());
                }
            }
            UpdateRequestProcessorChain processorChain = core.getUpdateProcessingChain(params.get(UpdateParams.UPDATE_CHAIN));
            wrappedReq = new LocalSolrQueryRequest(core, it.req.getParams());
            UpdateRequestProcessor processor = processorChain.createProcessor(wrappedReq, it.rsp);
            processor.processMergeIndexes(new MergeIndexesCommand(readers, it.req));
        } catch (Exception e) {
            // log and rethrow so that if the finally fails we don't lose the original problem
            log.error("ERROR executing merge:", e);
            throw e;
        } finally {
            for (RefCounted<SolrIndexSearcher> searcher : searchers) {
                if (searcher != null)
                    searcher.decref();
            }
            for (SolrCore solrCore : sourceCores) {
                if (solrCore != null)
                    solrCore.close();
            }
            IOUtils.closeWhileHandlingException(readersToBeClosed);
            Set<Map.Entry<Directory, Boolean>> entries = dirsToBeReleased.entrySet();
            for (Map.Entry<Directory, Boolean> entry : entries) {
                DirectoryFactory dirFactory = core.getDirectoryFactory();
                Directory dir = entry.getKey();
                boolean markAsDone = entry.getValue();
                if (markAsDone) {
                    dirFactory.doneWithDirectory(dir);
                }
                dirFactory.release(dir);
            }
            if (wrappedReq != null)
                wrappedReq.close();
            core.close();
        }
    }
}

Example 2 with DirectoryFactory

use of org.apache.solr.core.DirectoryFactory in project lucene-solr by apache.

the class HdfsWriteToMultipleCollectionsTest method test.

@Test
public void test() throws Exception {
    int docCount = random().nextInt(1313) + 1;
    int cnt = random().nextInt(4) + 1;
    for (int i = 0; i < cnt; i++) {
        createCollection(ACOLLECTION + i, 2, 2, 9);
    }
    for (int i = 0; i < cnt; i++) {
        waitForRecoveriesToFinish(ACOLLECTION + i, false);
    }
    List<CloudSolrClient> cloudClients = new ArrayList<>();
    List<StoppableIndexingThread> threads = new ArrayList<>();
    for (int i = 0; i < cnt; i++) {
        CloudSolrClient client = getCloudSolrClient(zkServer.getZkAddress());
        client.setDefaultCollection(ACOLLECTION + i);
        cloudClients.add(client);
        StoppableIndexingThread indexThread = new StoppableIndexingThread(null, client, "1", true, docCount, 1, true);
        threads.add(indexThread);
        indexThread.start();
    }
    int addCnt = 0;
    for (StoppableIndexingThread thread : threads) {
        thread.join();
        addCnt += thread.getNumAdds() - thread.getNumDeletes();
    }
    long collectionsCount = 0;
    for (CloudSolrClient client : cloudClients) {
        client.commit();
        collectionsCount += client.query(new SolrQuery("*:*")).getResults().getNumFound();
    }
    IOUtils.close(cloudClients);
    assertEquals(addCnt, collectionsCount);
    BlockCache lastBlockCache = null;
    // assert that we are using the block directory and that write and read caching are being used
    for (JettySolrRunner jetty : jettys) {
        CoreContainer cores = jetty.getCoreContainer();
        Collection<SolrCore> solrCores = cores.getCores();
        for (SolrCore core : solrCores) {
            if (core.getCoreDescriptor().getCloudDescriptor().getCollectionName().startsWith(ACOLLECTION)) {
                DirectoryFactory factory = core.getDirectoryFactory();
                assertTrue("Found: " + core.getDirectoryFactory().getClass().getName(), factory instanceof HdfsDirectoryFactory);
                Directory dir = factory.get(core.getDataDir(), null, null);
                try {
                    long dataDirSize = factory.size(dir);
                    FileSystem fileSystem = null;
                    fileSystem = FileSystem.newInstance(new Path(core.getDataDir()).toUri(), new Configuration());
                    long size = fileSystem.getContentSummary(new Path(core.getDataDir())).getLength();
                    assertEquals(size, dataDirSize);
                } finally {
                    core.getDirectoryFactory().release(dir);
                }
                RefCounted<IndexWriter> iwRef = core.getUpdateHandler().getSolrCoreState().getIndexWriter(core);
                try {
                    IndexWriter iw = iwRef.get();
                    NRTCachingDirectory directory = (NRTCachingDirectory) iw.getDirectory();
                    BlockDirectory blockDirectory = (BlockDirectory) directory.getDelegate();
                    assertTrue(blockDirectory.isBlockCacheReadEnabled());
                    // see SOLR-6424
                    assertFalse(blockDirectory.isBlockCacheWriteEnabled());
                    Cache cache = blockDirectory.getCache();
                    // we know it's a BlockDirectoryCache, but future proof
                    assertTrue(cache instanceof BlockDirectoryCache);
                    BlockCache blockCache = ((BlockDirectoryCache) cache).getBlockCache();
                    if (lastBlockCache != null) {
                        if (Boolean.getBoolean(SOLR_HDFS_BLOCKCACHE_GLOBAL)) {
                            assertEquals(lastBlockCache, blockCache);
                        } else {
                            assertNotSame(lastBlockCache, blockCache);
                        }
                    }
                    lastBlockCache = blockCache;
                } finally {
                    iwRef.decref();
                }
            }
        }
    }
}

Example 3 with DirectoryFactory

use of org.apache.solr.core.DirectoryFactory in project lucene-solr by apache.

the class CoreMergeIndexesAdminHandlerTest method testMergeIndexesCoreAdminHandler.

@Test
public void testMergeIndexesCoreAdminHandler() throws Exception {
    final File workDir = createTempDir().toFile();
    final CoreContainer cores = h.getCoreContainer();
    final CoreAdminHandler admin = new CoreAdminHandler(cores);
    try (SolrCore core = cores.getCore("collection1")) {
        DirectoryFactory df = core.getDirectoryFactory();
        FailingDirectoryFactory dirFactory = (FailingDirectoryFactory) df;
        try {
            dirFactory.fail = true;
            ignoreException(FAILING_MSG);
            SolrQueryResponse resp = new SolrQueryResponse();
            admin.handleRequestBody(req(CoreAdminParams.ACTION, CoreAdminParams.CoreAdminAction.MERGEINDEXES.toString(), CoreAdminParams.CORE, "collection1", CoreAdminParams.INDEX_DIR, workDir.getAbsolutePath()), resp);
            fail("exception expected");
        } catch (FailingDirectoryFactory.FailingDirectoryFactoryException e) {
        // expected if error handling properly
        } finally {
            unIgnoreException(FAILING_MSG);
        }
        dirFactory.fail = false;
    }
}