Examples with MergeScheduler org.apache.lucene.index.MergeScheduler used on opensource projects

Example 1 with MergeScheduler

use of org.apache.lucene.index.MergeScheduler in project lucene-solr by apache.

the class TestUtil method reduceOpenFiles.

/** just tries to configure things to keep the open file
   * count lowish */
public static void reduceOpenFiles(IndexWriter w) {
    // keep number of open files lowish
    MergePolicy mp = w.getConfig().getMergePolicy();
    mp.setNoCFSRatio(1.0);
    if (mp instanceof LogMergePolicy) {
        LogMergePolicy lmp = (LogMergePolicy) mp;
        lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
    } else if (mp instanceof TieredMergePolicy) {
        TieredMergePolicy tmp = (TieredMergePolicy) mp;
        tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
        tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
    }
    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
        // wtf... shouldnt it be even lower since it's 1 by default?!?!
        ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);
    }
}

Example 2 with MergeScheduler

use of org.apache.lucene.index.MergeScheduler in project lucene-solr by apache.

the class TestIDVersionPostingsFormat method testMoreThanOneDocPerIDTwoSegments.

public void testMoreThanOneDocPerIDTwoSegments() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
    iwc.setCodec(TestUtil.alwaysPostingsFormat(new IDVersionPostingsFormat()));
    iwc.setMergePolicy(new TieredMergePolicy());
    MergeScheduler ms = iwc.getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
        iwc.setMergeScheduler(new ConcurrentMergeScheduler() {

            @Override
            protected void handleMergeException(Directory dir, Throwable exc) {
                assertTrue(exc instanceof IllegalArgumentException);
            }
        });
    }
    IndexWriter w = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(makeIDField("id", 17));
    w.addDocument(doc);
    w.commit();
    doc = new Document();
    doc.add(makeIDField("id", 17));
    try {
        w.addDocument(doc);
        w.commit();
        w.forceMerge(1);
        fail("didn't hit exception");
    } catch (IllegalArgumentException iae) {
    // expected: SMS will hit this
    } catch (IOException | IllegalStateException exc) {
        // expected
        assertTrue(exc.getCause() instanceof IllegalArgumentException);
    }
    w.rollback();
    dir.close();
}

Example 3 with MergeScheduler

use of org.apache.lucene.index.MergeScheduler in project lucene-solr by apache.

the class SolrIndexConfig method toIndexWriterConfig.

public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException {
    IndexSchema schema = core.getLatestSchema();
    IndexWriterConfig iwc = new IndexWriterConfig(new DelayedSchemaAnalyzer(core));
    if (maxBufferedDocs != -1)
        iwc.setMaxBufferedDocs(maxBufferedDocs);
    if (ramBufferSizeMB != -1)
        iwc.setRAMBufferSizeMB(ramBufferSizeMB);
    iwc.setSimilarity(schema.getSimilarity());
    MergePolicy mergePolicy = buildMergePolicy(schema);
    iwc.setMergePolicy(mergePolicy);
    MergeScheduler mergeScheduler = buildMergeScheduler(schema);
    iwc.setMergeScheduler(mergeScheduler);
    iwc.setInfoStream(infoStream);
    if (mergePolicy instanceof SortingMergePolicy) {
        Sort indexSort = ((SortingMergePolicy) mergePolicy).getSort();
        iwc.setIndexSort(indexSort);
    }
    iwc.setUseCompoundFile(useCompoundFile);
    if (mergedSegmentWarmerInfo != null) {
        // TODO: add infostream -> normal logging system (there is an issue somewhere)
        IndexReaderWarmer warmer = schema.getResourceLoader().newInstance(mergedSegmentWarmerInfo.className, IndexReaderWarmer.class, null, new Class[] { InfoStream.class }, new Object[] { iwc.getInfoStream() });
        iwc.setMergedSegmentWarmer(warmer);
    }
    return iwc;
}

Example 4 with MergeScheduler

use of org.apache.lucene.index.MergeScheduler in project lucene-solr by apache.

the class SolrIndexConfig method buildMergeScheduler.

private MergeScheduler buildMergeScheduler(IndexSchema schema) {
    String msClassName = mergeSchedulerInfo == null ? SolrIndexConfig.DEFAULT_MERGE_SCHEDULER_CLASSNAME : mergeSchedulerInfo.className;
    MergeScheduler scheduler = schema.getResourceLoader().newInstance(msClassName, MergeScheduler.class);
    if (mergeSchedulerInfo != null) {
        // if someone has them configured.
        if (scheduler instanceof ConcurrentMergeScheduler) {
            NamedList args = mergeSchedulerInfo.initArgs.clone();
            Integer maxMergeCount = (Integer) args.remove("maxMergeCount");
            if (maxMergeCount == null) {
                maxMergeCount = ((ConcurrentMergeScheduler) scheduler).getMaxMergeCount();
            }
            Integer maxThreadCount = (Integer) args.remove("maxThreadCount");
            if (maxThreadCount == null) {
                maxThreadCount = ((ConcurrentMergeScheduler) scheduler).getMaxThreadCount();
            }
            ((ConcurrentMergeScheduler) scheduler).setMaxMergesAndThreads(maxMergeCount, maxThreadCount);
            SolrPluginUtils.invokeSetters(scheduler, args);
        } else {
            SolrPluginUtils.invokeSetters(scheduler, mergeSchedulerInfo.initArgs);
        }
    }
    return scheduler;
}