Examples with MergePolicy org.apache.lucene.index.MergePolicy used on opensource projects

Example 6 with MergePolicy

use of org.apache.lucene.index.MergePolicy in project lucene-solr by apache.

the class TestUtil method reduceOpenFiles.

/** just tries to configure things to keep the open file
   * count lowish */
public static void reduceOpenFiles(IndexWriter w) {
    // keep number of open files lowish
    MergePolicy mp = w.getConfig().getMergePolicy();
    mp.setNoCFSRatio(1.0);
    if (mp instanceof LogMergePolicy) {
        LogMergePolicy lmp = (LogMergePolicy) mp;
        lmp.setMergeFactor(Math.min(5, lmp.getMergeFactor()));
    } else if (mp instanceof TieredMergePolicy) {
        TieredMergePolicy tmp = (TieredMergePolicy) mp;
        tmp.setMaxMergeAtOnce(Math.min(5, tmp.getMaxMergeAtOnce()));
        tmp.setSegmentsPerTier(Math.min(5, tmp.getSegmentsPerTier()));
    }
    MergeScheduler ms = w.getConfig().getMergeScheduler();
    if (ms instanceof ConcurrentMergeScheduler) {
        // wtf... shouldnt it be even lower since it's 1 by default?!?!
        ((ConcurrentMergeScheduler) ms).setMaxMergesAndThreads(3, 2);
    }
}

Example 7 with MergePolicy

use of org.apache.lucene.index.MergePolicy in project elasticsearch by elastic.

the class InternalEngine method getIndexWriterConfig.

private IndexWriterConfig getIndexWriterConfig(boolean create) {
    final IndexWriterConfig iwc = new IndexWriterConfig(engineConfig.getAnalyzer());
    // we by default don't commit on close
    iwc.setCommitOnClose(false);
    iwc.setOpenMode(create ? IndexWriterConfig.OpenMode.CREATE : IndexWriterConfig.OpenMode.APPEND);
    iwc.setIndexDeletionPolicy(deletionPolicy);
    // with tests.verbose, lucene sets this up: plumb to align with filesystem stream
    boolean verbose = false;
    try {
        verbose = Boolean.parseBoolean(System.getProperty("tests.verbose"));
    } catch (Exception ignore) {
    }
    iwc.setInfoStream(verbose ? InfoStream.getDefault() : new LoggerInfoStream(logger));
    iwc.setMergeScheduler(mergeScheduler);
    MergePolicy mergePolicy = config().getMergePolicy();
    // Give us the opportunity to upgrade old segments while performing
    // background merges
    mergePolicy = new ElasticsearchMergePolicy(mergePolicy);
    iwc.setMergePolicy(mergePolicy);
    iwc.setSimilarity(engineConfig.getSimilarity());
    iwc.setRAMBufferSizeMB(engineConfig.getIndexingBufferSize().getMbFrac());
    iwc.setCodec(engineConfig.getCodec());
    // always use compound on flush - reduces # of file-handles on refresh
    iwc.setUseCompoundFile(true);
    return iwc;
}

Example 8 with MergePolicy

use of org.apache.lucene.index.MergePolicy in project lucene-solr by apache.

the class WrapperMergePolicyFactoryTest method testProperlyInitializesWrappedMergePolicy.

public void testProperlyInitializesWrappedMergePolicy() {
    final TieredMergePolicy defaultTMP = new TieredMergePolicy();
    final int testMaxMergeAtOnce = defaultTMP.getMaxMergeAtOnce() * 2;
    final double testMaxMergedSegmentMB = defaultTMP.getMaxMergedSegmentMB() * 10;
    final MergePolicyFactoryArgs args = new MergePolicyFactoryArgs();
    args.add(WrapperMergePolicyFactory.WRAPPED_PREFIX, "test");
    args.add("test.class", TieredMergePolicyFactory.class.getName());
    args.add("test.maxMergeAtOnce", testMaxMergeAtOnce);
    args.add("test.maxMergedSegmentMB", testMaxMergedSegmentMB);
    MergePolicyFactory mpf = new DefaultingWrapperMergePolicyFactory(resourceLoader, args, null) {

        @Override
        protected MergePolicy getDefaultWrappedMergePolicy() {
            throw new IllegalStateException("Should not have reached here!");
        }
    };
    final MergePolicy mp = mpf.getMergePolicy();
    assertSame(mp.getClass(), TieredMergePolicy.class);
    final TieredMergePolicy tmp = (TieredMergePolicy) mp;
    assertEquals("maxMergeAtOnce", testMaxMergeAtOnce, tmp.getMaxMergeAtOnce());
    assertEquals("maxMergedSegmentMB", testMaxMergedSegmentMB, tmp.getMaxMergedSegmentMB(), 0.0d);
}

Example 9 with MergePolicy

use of org.apache.lucene.index.MergePolicy in project lucene-solr by apache.

the class SolrIndexConfig method toIndexWriterConfig.

public IndexWriterConfig toIndexWriterConfig(SolrCore core) throws IOException {
    IndexSchema schema = core.getLatestSchema();
    IndexWriterConfig iwc = new IndexWriterConfig(new DelayedSchemaAnalyzer(core));
    if (maxBufferedDocs != -1)
        iwc.setMaxBufferedDocs(maxBufferedDocs);
    if (ramBufferSizeMB != -1)
        iwc.setRAMBufferSizeMB(ramBufferSizeMB);
    iwc.setSimilarity(schema.getSimilarity());
    MergePolicy mergePolicy = buildMergePolicy(schema);
    iwc.setMergePolicy(mergePolicy);
    MergeScheduler mergeScheduler = buildMergeScheduler(schema);
    iwc.setMergeScheduler(mergeScheduler);
    iwc.setInfoStream(infoStream);
    if (mergePolicy instanceof SortingMergePolicy) {
        Sort indexSort = ((SortingMergePolicy) mergePolicy).getSort();
        iwc.setIndexSort(indexSort);
    }
    iwc.setUseCompoundFile(useCompoundFile);
    if (mergedSegmentWarmerInfo != null) {
        // TODO: add infostream -> normal logging system (there is an issue somewhere)
        IndexReaderWarmer warmer = schema.getResourceLoader().newInstance(mergedSegmentWarmerInfo.className, IndexReaderWarmer.class, null, new Class[] { InfoStream.class }, new Object[] { iwc.getInfoStream() });
        iwc.setMergedSegmentWarmer(warmer);
    }
    return iwc;
}

Example 10 with MergePolicy

use of org.apache.lucene.index.MergePolicy in project lucene-solr by apache.

the class WrapperMergePolicyFactory method getMergePolicy.

/** Returns a wrapping {@link MergePolicy} with its set parameters configured. */
@Override
public final MergePolicy getMergePolicy() {
    final MergePolicy wrappedMP = getWrappedMergePolicy();
    final MergePolicy mp = getMergePolicyInstance(wrappedMP);
    args.invokeSetters(mp);
    return mp;
}