Examples with LogByteSizeMergePolicy org.apache.lucene.index.LogByteSizeMergePolicy used on opensource projects

Example 6 with LogByteSizeMergePolicy

use of org.apache.lucene.index.LogByteSizeMergePolicy in project neo4j by neo4j.

the class IndexWriterConfigs method standard.

public static IndexWriterConfig standard() {
    IndexWriterConfig writerConfig = new IndexWriterConfig(LuceneDataSource.KEYWORD_ANALYZER);
    writerConfig.setMaxBufferedDocs(MAX_BUFFERED_DOCS);
    writerConfig.setMaxBufferedDeleteTerms(MAX_BUFFERED_DELETE_TERMS);
    writerConfig.setIndexDeletionPolicy(new MultipleBackupDeletionPolicy());
    writerConfig.setUseCompoundFile(true);
    writerConfig.setRAMBufferSizeMB(STANDARD_RAM_BUFFER_SIZE_MB);
    writerConfig.setCodec(new Lucene54Codec() {

        @Override
        public PostingsFormat getPostingsFormatForField(String field) {
            PostingsFormat postingFormat = super.getPostingsFormatForField(field);
            return CODEC_BLOCK_TREE_ORDS_POSTING_FORMAT ? blockTreeOrdsPostingsFormat : postingFormat;
        }
    });
    LogByteSizeMergePolicy mergePolicy = new LogByteSizeMergePolicy();
    mergePolicy.setNoCFSRatio(MERGE_POLICY_NO_CFS_RATIO);
    mergePolicy.setMinMergeMB(MERGE_POLICY_MIN_MERGE_MB);
    mergePolicy.setMergeFactor(MERGE_POLICY_MERGE_FACTOR);
    writerConfig.setMergePolicy(mergePolicy);
    return writerConfig;
}

Example 7 with LogByteSizeMergePolicy

use of org.apache.lucene.index.LogByteSizeMergePolicy in project zm-mailbox by Zimbra.

the class LuceneIndex method getWriterConfig.

private IndexWriterConfig getWriterConfig() {
    IndexWriterConfig config = new IndexWriterConfig(VERSION, mailbox.index.getAnalyzer());
    config.setMergeScheduler(new MergeScheduler());
    config.setMaxBufferedDocs(LC.zimbra_index_lucene_max_buffered_docs.intValue());
    config.setRAMBufferSizeMB(LC.zimbra_index_lucene_ram_buffer_size_kb.intValue() / 1024.0);
    if (LC.zimbra_index_lucene_merge_policy.booleanValue()) {
        LogDocMergePolicy policy = new LogDocMergePolicy();
        config.setMergePolicy(policy);
        policy.setUseCompoundFile(LC.zimbra_index_lucene_use_compound_file.booleanValue());
        policy.setMergeFactor(LC.zimbra_index_lucene_merge_factor.intValue());
        policy.setMinMergeDocs(LC.zimbra_index_lucene_min_merge.intValue());
        if (LC.zimbra_index_lucene_max_merge.intValue() != Integer.MAX_VALUE) {
            policy.setMaxMergeDocs(LC.zimbra_index_lucene_max_merge.intValue());
        }
    } else {
        LogByteSizeMergePolicy policy = new LogByteSizeMergePolicy();
        config.setMergePolicy(policy);
        policy.setUseCompoundFile(LC.zimbra_index_lucene_use_compound_file.booleanValue());
        policy.setMergeFactor(LC.zimbra_index_lucene_merge_factor.intValue());
        policy.setMinMergeMB(LC.zimbra_index_lucene_min_merge.intValue() / 1024.0);
        if (LC.zimbra_index_lucene_max_merge.intValue() != Integer.MAX_VALUE) {
            policy.setMaxMergeMB(LC.zimbra_index_lucene_max_merge.intValue() / 1024.0);
        }
    }
    return config;
}

Example 8 with LogByteSizeMergePolicy

use of org.apache.lucene.index.LogByteSizeMergePolicy in project jackrabbit by apache.

the class AbstractIndex method getIndexWriter.

/**
     * Returns an <code>IndexWriter</code> on this index.
     * @return an <code>IndexWriter</code> on this index.
     * @throws IOException if the writer cannot be obtained.
     */
protected synchronized IndexWriter getIndexWriter() throws IOException {
    if (indexReader != null) {
        indexReader.close();
        log.debug("closing IndexReader.");
        indexReader = null;
    }
    if (indexWriter == null) {
        IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_36, analyzer);
        config.setSimilarity(similarity);
        LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
        mergePolicy.setUseCompoundFile(useCompoundFile);
        mergePolicy.setNoCFSRatio(1.0);
        config.setMergePolicy(mergePolicy);
        indexWriter = new IndexWriter(getDirectory(), config);
        indexWriter.setInfoStream(STREAM_LOGGER);
    }
    return indexWriter;
}

Example 9 with LogByteSizeMergePolicy

use of org.apache.lucene.index.LogByteSizeMergePolicy in project jackrabbit by apache.

the class IndexMigration method migrate.

/**
     * Checks if the given <code>index</code> needs to be migrated.
     *
     * @param index the index to check and migration if needed.
     * @param directoryManager the directory manager.
     * @param oldSeparatorChar the old separator char that needs to be replaced.
     * @throws IOException if an error occurs while migrating the index.
     */
public static void migrate(PersistentIndex index, DirectoryManager directoryManager, char oldSeparatorChar) throws IOException {
    Directory indexDir = index.getDirectory();
    log.debug("Checking {} ...", indexDir);
    ReadOnlyIndexReader reader = index.getReadOnlyIndexReader();
    try {
        if (IndexFormatVersion.getVersion(reader).getVersion() >= IndexFormatVersion.V3.getVersion()) {
            // index was created with Jackrabbit 1.5 or higher
            // no need for migration
            log.debug("IndexFormatVersion >= V3, no migration needed");
            return;
        }
        // assert: there is at least one node in the index, otherwise the
        //         index format version would be at least V3
        TermEnum terms = reader.terms(new Term(FieldNames.PROPERTIES, ""));
        try {
            Term t = terms.term();
            if (t.text().indexOf(oldSeparatorChar) == -1) {
                log.debug("Index already migrated");
                return;
            }
        } finally {
            terms.close();
        }
    } finally {
        reader.release();
        index.releaseWriterAndReaders();
    }
    // if we get here then the index must be migrated
    log.debug("Index requires migration {}", indexDir);
    String migrationName = index.getName() + "_v36";
    if (directoryManager.hasDirectory(migrationName)) {
        directoryManager.delete(migrationName);
    }
    Directory migrationDir = directoryManager.getDirectory(migrationName);
    final IndexWriterConfig c = new IndexWriterConfig(Version.LUCENE_36, new JackrabbitAnalyzer());
    c.setMergePolicy(new UpgradeIndexMergePolicy(new LogByteSizeMergePolicy()));
    c.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
    try {
        IndexWriter writer = new IndexWriter(migrationDir, c);
        try {
            IndexReader r = new MigrationIndexReader(IndexReader.open(index.getDirectory()), oldSeparatorChar);
            try {
                writer.addIndexes(r);
                writer.forceMerge(1);
                writer.close();
            } finally {
                r.close();
            }
        } finally {
            writer.close();
        }
    } finally {
        migrationDir.close();
    }
    directoryManager.delete(index.getName());
    if (!directoryManager.rename(migrationName, index.getName())) {
        throw new IOException("failed to move migrated directory " + migrationDir);
    }
    log.info("Migrated " + index.getName());
}

Example 10 with LogByteSizeMergePolicy

use of org.apache.lucene.index.LogByteSizeMergePolicy in project lucene-solr by apache.

the class TestDirectoryTaxonomyReader method testOpenIfChangedMergedSegment.

@Test
public void testOpenIfChangedMergedSegment() throws Exception {
    // test openIfChanged() when all index segments were merged - used to be
    // a bug in ParentArray, caught by testOpenIfChangedManySegments - only
    // this test is not random
    Directory dir = newDirectory();
    // hold onto IW to forceMerge
    // note how we don't close it, since DTW will close it.
    final IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(new LogByteSizeMergePolicy()));
    DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {

        @Override
        protected IndexWriter openIndexWriter(Directory directory, IndexWriterConfig config) throws IOException {
            return iw;
        }
    };
    TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
    assertEquals(1, reader.getSize());
    assertEquals(1, reader.getParallelTaxonomyArrays().parents().length);
    // add category and call forceMerge -- this should flush IW and merge segments down to 1
    // in ParentArray.initFromReader, this used to fail assuming there are no parents.
    writer.addCategory(new FacetLabel("1"));
    iw.forceMerge(1);
    // now calling openIfChanged should trip on the bug
    TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader);
    assertNotNull(newtr);
    reader.close();
    reader = newtr;
    assertEquals(2, reader.getSize());
    assertEquals(2, reader.getParallelTaxonomyArrays().parents().length);
    reader.close();
    writer.close();
    dir.close();
}