Examples with MockDirectoryWrapper org.apache.lucene.store.MockDirectoryWrapper used on opensource projects

Example 56 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project Anserini by castorini.

the class BaseFeatureExtractorTest method setUp.

/**
 * MUST call super
 * constructs the necessary rerankers and extractorchains
 * @throws Exception
 */
@Override
@Before
public void setUp() throws Exception {
    super.setUp();
    // Use a RAMDirectory instead of MemoryIndex because we might test with multiple documents
    DIRECTORY = new MockDirectoryWrapper(new Random(), new RAMDirectory());
    testWriter = new IndexWriter(DIRECTORY, new IndexWriterConfig(TEST_ANALYZER));
}

Example 57 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project Anserini by castorini.

the class TweetSearch method setUp.

/**
 * MUST call super
 * constructs the necessary rerankers and extractorchains
 * @throws Exception
 */
@Override
@Before
public void setUp() throws Exception {
    super.setUp();
    // Use a RAMDirectory instead of MemoryIndex because we might test with multiple documents
    DIRECTORY = new MockDirectoryWrapper(new Random(), new RAMDirectory());
    testWriter = new IndexWriter(DIRECTORY, new IndexWriterConfig(TEST_ANALYZER));
}

Example 58 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project elasticsearch by elastic.

the class LuceneTests method testPruneUnreferencedFiles.

public void testPruneUnreferencedFiles() throws IOException {
    MockDirectoryWrapper dir = newMockDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig();
    iwc.setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE);
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    iwc.setMaxBufferedDocs(2);
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new TextField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();
    doc = new Document();
    doc.add(new TextField("id", "2", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    doc = new Document();
    doc.add(new TextField("id", "3", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();
    SegmentInfos segmentCommitInfos = Lucene.readSegmentInfos(dir);
    doc = new Document();
    doc.add(new TextField("id", "4", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.deleteDocuments(new Term("id", "2"));
    writer.commit();
    DirectoryReader open = DirectoryReader.open(writer);
    assertEquals(3, open.numDocs());
    assertEquals(1, open.numDeletedDocs());
    assertEquals(4, open.maxDoc());
    open.close();
    writer.close();
    SegmentInfos si = Lucene.pruneUnreferencedFiles(segmentCommitInfos.getSegmentsFileName(), dir);
    assertEquals(si.getSegmentsFileName(), segmentCommitInfos.getSegmentsFileName());
    open = DirectoryReader.open(dir);
    assertEquals(3, open.numDocs());
    assertEquals(0, open.numDeletedDocs());
    assertEquals(3, open.maxDoc());
    IndexSearcher s = new IndexSearcher(open);
    assertEquals(s.search(new TermQuery(new Term("id", "1")), 1).totalHits, 1);
    assertEquals(s.search(new TermQuery(new Term("id", "2")), 1).totalHits, 1);
    assertEquals(s.search(new TermQuery(new Term("id", "3")), 1).totalHits, 1);
    assertEquals(s.search(new TermQuery(new Term("id", "4")), 1).totalHits, 0);
    for (String file : dir.listAll()) {
        assertFalse("unexpected file: " + file, file.equals("segments_3") || file.startsWith("_2"));
    }
    open.close();
    dir.close();
}

Example 59 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project elasticsearch by elastic.

the class LuceneTests method testFiles.

public void testFiles() throws IOException {
    MockDirectoryWrapper dir = newMockDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    iwc.setMaxBufferedDocs(2);
    iwc.setUseCompoundFile(true);
    IndexWriter writer = new IndexWriter(dir, iwc);
    Document doc = new Document();
    doc.add(new TextField("id", "1", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();
    Set<String> files = new HashSet<>();
    for (String f : Lucene.files(Lucene.readSegmentInfos(dir))) {
        files.add(f);
    }
    final boolean simpleTextCFS = files.contains("_0.scf");
    assertTrue(files.toString(), files.contains("segments_1"));
    if (simpleTextCFS) {
        assertFalse(files.toString(), files.contains("_0.cfs"));
        assertFalse(files.toString(), files.contains("_0.cfe"));
    } else {
        assertTrue(files.toString(), files.contains("_0.cfs"));
        assertTrue(files.toString(), files.contains("_0.cfe"));
    }
    assertTrue(files.toString(), files.contains("_0.si"));
    doc = new Document();
    doc.add(new TextField("id", "2", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    doc = new Document();
    doc.add(new TextField("id", "3", random().nextBoolean() ? Field.Store.YES : Field.Store.NO));
    writer.addDocument(doc);
    writer.commit();
    files.clear();
    for (String f : Lucene.files(Lucene.readSegmentInfos(dir))) {
        files.add(f);
    }
    assertFalse(files.toString(), files.contains("segments_1"));
    assertTrue(files.toString(), files.contains("segments_2"));
    if (simpleTextCFS) {
        assertFalse(files.toString(), files.contains("_0.cfs"));
        assertFalse(files.toString(), files.contains("_0.cfe"));
    } else {
        assertTrue(files.toString(), files.contains("_0.cfs"));
        assertTrue(files.toString(), files.contains("_0.cfe"));
    }
    assertTrue(files.toString(), files.contains("_0.si"));
    if (simpleTextCFS) {
        assertFalse(files.toString(), files.contains("_1.cfs"));
        assertFalse(files.toString(), files.contains("_1.cfe"));
    } else {
        assertTrue(files.toString(), files.contains("_1.cfs"));
        assertTrue(files.toString(), files.contains("_1.cfe"));
    }
    assertTrue(files.toString(), files.contains("_1.si"));
    writer.close();
    dir.close();
}

Example 60 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project elasticsearch by elastic.

the class InternalEngineTests method testTranslogReplayWithFailure.

public void testTranslogReplayWithFailure() throws IOException {
    final int numDocs = randomIntBetween(1, 10);
    for (int i = 0; i < numDocs; i++) {
        ParsedDocument doc = testParsedDocument(Integer.toString(i), "test", null, testDocument(), new BytesArray("{}"), null);
        Engine.Index firstIndexRequest = new Engine.Index(newUid(doc), doc, SequenceNumbersService.UNASSIGNED_SEQ_NO, 0, Versions.MATCH_DELETED, VersionType.INTERNAL, PRIMARY, System.nanoTime(), -1, false);
        Engine.IndexResult indexResult = engine.index(firstIndexRequest);
        assertThat(indexResult.getVersion(), equalTo(1L));
    }
    engine.refresh("test");
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
    engine.close();
    final MockDirectoryWrapper directory = DirectoryUtils.getLeaf(store.directory(), MockDirectoryWrapper.class);
    if (directory != null) {
        // since we rollback the IW we are writing the same segment files again after starting IW but MDW prevents
        // this so we have to disable the check explicitly
        boolean started = false;
        final int numIters = randomIntBetween(10, 20);
        for (int i = 0; i < numIters; i++) {
            directory.setRandomIOExceptionRateOnOpen(randomDouble());
            directory.setRandomIOExceptionRate(randomDouble());
            directory.setFailOnOpenInput(randomBoolean());
            directory.setAllowRandomFileNotFoundException(randomBoolean());
            try {
                engine = createEngine(store, primaryTranslogDir);
                started = true;
                break;
            } catch (EngineException | IOException e) {
            }
        }
        directory.setRandomIOExceptionRateOnOpen(0.0);
        directory.setRandomIOExceptionRate(0.0);
        directory.setFailOnOpenInput(false);
        directory.setAllowRandomFileNotFoundException(false);
        if (started == false) {
            engine = createEngine(store, primaryTranslogDir);
        }
    } else {
        // no mock directory, no fun.
        engine = createEngine(store, primaryTranslogDir);
    }
    try (Engine.Searcher searcher = engine.acquireSearcher("test")) {
        TopDocs topDocs = searcher.searcher().search(new MatchAllDocsQuery(), randomIntBetween(numDocs, numDocs + 10));
        assertThat(topDocs.totalHits, equalTo(numDocs));
    }
}