use of org.apache.lucene.index.LogByteSizeMergePolicy in project neo4j by neo4j.
the class IndexWriterConfigs method standard.
public static IndexWriterConfig standard() {
IndexWriterConfig writerConfig = new IndexWriterConfig(LuceneDataSource.KEYWORD_ANALYZER);
writerConfig.setMaxBufferedDocs(MAX_BUFFERED_DOCS);
writerConfig.setMaxBufferedDeleteTerms(MAX_BUFFERED_DELETE_TERMS);
writerConfig.setIndexDeletionPolicy(new MultipleBackupDeletionPolicy());
writerConfig.setUseCompoundFile(true);
writerConfig.setRAMBufferSizeMB(STANDARD_RAM_BUFFER_SIZE_MB);
writerConfig.setCodec(new Lucene54Codec() {
@Override
public PostingsFormat getPostingsFormatForField(String field) {
PostingsFormat postingFormat = super.getPostingsFormatForField(field);
return CODEC_BLOCK_TREE_ORDS_POSTING_FORMAT ? blockTreeOrdsPostingsFormat : postingFormat;
}
});
LogByteSizeMergePolicy mergePolicy = new LogByteSizeMergePolicy();
mergePolicy.setNoCFSRatio(MERGE_POLICY_NO_CFS_RATIO);
mergePolicy.setMinMergeMB(MERGE_POLICY_MIN_MERGE_MB);
mergePolicy.setMergeFactor(MERGE_POLICY_MERGE_FACTOR);
writerConfig.setMergePolicy(mergePolicy);
return writerConfig;
}
use of org.apache.lucene.index.LogByteSizeMergePolicy in project zm-mailbox by Zimbra.
the class LuceneIndex method getWriterConfig.
private IndexWriterConfig getWriterConfig() {
IndexWriterConfig config = new IndexWriterConfig(VERSION, mailbox.index.getAnalyzer());
config.setMergeScheduler(new MergeScheduler());
config.setMaxBufferedDocs(LC.zimbra_index_lucene_max_buffered_docs.intValue());
config.setRAMBufferSizeMB(LC.zimbra_index_lucene_ram_buffer_size_kb.intValue() / 1024.0);
if (LC.zimbra_index_lucene_merge_policy.booleanValue()) {
LogDocMergePolicy policy = new LogDocMergePolicy();
config.setMergePolicy(policy);
policy.setUseCompoundFile(LC.zimbra_index_lucene_use_compound_file.booleanValue());
policy.setMergeFactor(LC.zimbra_index_lucene_merge_factor.intValue());
policy.setMinMergeDocs(LC.zimbra_index_lucene_min_merge.intValue());
if (LC.zimbra_index_lucene_max_merge.intValue() != Integer.MAX_VALUE) {
policy.setMaxMergeDocs(LC.zimbra_index_lucene_max_merge.intValue());
}
} else {
LogByteSizeMergePolicy policy = new LogByteSizeMergePolicy();
config.setMergePolicy(policy);
policy.setUseCompoundFile(LC.zimbra_index_lucene_use_compound_file.booleanValue());
policy.setMergeFactor(LC.zimbra_index_lucene_merge_factor.intValue());
policy.setMinMergeMB(LC.zimbra_index_lucene_min_merge.intValue() / 1024.0);
if (LC.zimbra_index_lucene_max_merge.intValue() != Integer.MAX_VALUE) {
policy.setMaxMergeMB(LC.zimbra_index_lucene_max_merge.intValue() / 1024.0);
}
}
return config;
}
use of org.apache.lucene.index.LogByteSizeMergePolicy in project jackrabbit by apache.
the class AbstractIndex method getIndexWriter.
/**
* Returns an <code>IndexWriter</code> on this index.
* @return an <code>IndexWriter</code> on this index.
* @throws IOException if the writer cannot be obtained.
*/
protected synchronized IndexWriter getIndexWriter() throws IOException {
if (indexReader != null) {
indexReader.close();
log.debug("closing IndexReader.");
indexReader = null;
}
if (indexWriter == null) {
IndexWriterConfig config = new IndexWriterConfig(Version.LUCENE_36, analyzer);
config.setSimilarity(similarity);
LogMergePolicy mergePolicy = new LogByteSizeMergePolicy();
mergePolicy.setUseCompoundFile(useCompoundFile);
mergePolicy.setNoCFSRatio(1.0);
config.setMergePolicy(mergePolicy);
indexWriter = new IndexWriter(getDirectory(), config);
indexWriter.setInfoStream(STREAM_LOGGER);
}
return indexWriter;
}
use of org.apache.lucene.index.LogByteSizeMergePolicy in project jackrabbit by apache.
the class IndexMigration method migrate.
/**
* Checks if the given <code>index</code> needs to be migrated.
*
* @param index the index to check and migration if needed.
* @param directoryManager the directory manager.
* @param oldSeparatorChar the old separator char that needs to be replaced.
* @throws IOException if an error occurs while migrating the index.
*/
public static void migrate(PersistentIndex index, DirectoryManager directoryManager, char oldSeparatorChar) throws IOException {
Directory indexDir = index.getDirectory();
log.debug("Checking {} ...", indexDir);
ReadOnlyIndexReader reader = index.getReadOnlyIndexReader();
try {
if (IndexFormatVersion.getVersion(reader).getVersion() >= IndexFormatVersion.V3.getVersion()) {
// index was created with Jackrabbit 1.5 or higher
// no need for migration
log.debug("IndexFormatVersion >= V3, no migration needed");
return;
}
// assert: there is at least one node in the index, otherwise the
// index format version would be at least V3
TermEnum terms = reader.terms(new Term(FieldNames.PROPERTIES, ""));
try {
Term t = terms.term();
if (t.text().indexOf(oldSeparatorChar) == -1) {
log.debug("Index already migrated");
return;
}
} finally {
terms.close();
}
} finally {
reader.release();
index.releaseWriterAndReaders();
}
// if we get here then the index must be migrated
log.debug("Index requires migration {}", indexDir);
String migrationName = index.getName() + "_v36";
if (directoryManager.hasDirectory(migrationName)) {
directoryManager.delete(migrationName);
}
Directory migrationDir = directoryManager.getDirectory(migrationName);
final IndexWriterConfig c = new IndexWriterConfig(Version.LUCENE_36, new JackrabbitAnalyzer());
c.setMergePolicy(new UpgradeIndexMergePolicy(new LogByteSizeMergePolicy()));
c.setIndexDeletionPolicy(new KeepOnlyLastCommitDeletionPolicy());
try {
IndexWriter writer = new IndexWriter(migrationDir, c);
try {
IndexReader r = new MigrationIndexReader(IndexReader.open(index.getDirectory()), oldSeparatorChar);
try {
writer.addIndexes(r);
writer.forceMerge(1);
writer.close();
} finally {
r.close();
}
} finally {
writer.close();
}
} finally {
migrationDir.close();
}
directoryManager.delete(index.getName());
if (!directoryManager.rename(migrationName, index.getName())) {
throw new IOException("failed to move migrated directory " + migrationDir);
}
log.info("Migrated " + index.getName());
}
use of org.apache.lucene.index.LogByteSizeMergePolicy in project lucene-solr by apache.
the class TestDirectoryTaxonomyReader method testOpenIfChangedMergedSegment.
@Test
public void testOpenIfChangedMergedSegment() throws Exception {
// test openIfChanged() when all index segments were merged - used to be
// a bug in ParentArray, caught by testOpenIfChangedManySegments - only
// this test is not random
Directory dir = newDirectory();
// hold onto IW to forceMerge
// note how we don't close it, since DTW will close it.
final IndexWriter iw = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMergePolicy(new LogByteSizeMergePolicy()));
DirectoryTaxonomyWriter writer = new DirectoryTaxonomyWriter(dir) {
@Override
protected IndexWriter openIndexWriter(Directory directory, IndexWriterConfig config) throws IOException {
return iw;
}
};
TaxonomyReader reader = new DirectoryTaxonomyReader(writer);
assertEquals(1, reader.getSize());
assertEquals(1, reader.getParallelTaxonomyArrays().parents().length);
// add category and call forceMerge -- this should flush IW and merge segments down to 1
// in ParentArray.initFromReader, this used to fail assuming there are no parents.
writer.addCategory(new FacetLabel("1"));
iw.forceMerge(1);
// now calling openIfChanged should trip on the bug
TaxonomyReader newtr = TaxonomyReader.openIfChanged(reader);
assertNotNull(newtr);
reader.close();
reader = newtr;
assertEquals(2, reader.getSize());
assertEquals(2, reader.getParallelTaxonomyArrays().parents().length);
reader.close();
writer.close();
dir.close();
}
Aggregations