Examples with MockDirectoryWrapper org.apache.lucene.store.MockDirectoryWrapper used on opensource projects

Example 6 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class TestIndexWriterCommit method testCommitOnCloseDiskUsage.

/*
   * Verify that a writer with "commit on close" indeed
   * cleans up the temp segments created after opening
   * that are not referenced by the starting segments
   * file.  We check this by using MockDirectoryWrapper to
   * measure max temp disk space used.
   */
public void testCommitOnCloseDiskUsage() throws IOException {
    // MemoryCodec, since it uses FST, is not necessarily
    // "additive", ie if you add up N small FSTs, then merge
    // them, the merged result can easily be larger than the
    // sum because the merged FST may use array encoding for
    // some arcs (which uses more space):
    final String idFormat = TestUtil.getPostingsFormat("id");
    final String contentFormat = TestUtil.getPostingsFormat("content");
    assumeFalse("This test cannot run with Memory codec", idFormat.equals("Memory") || contentFormat.equals("Memory"));
    MockDirectoryWrapper dir = newMockDirectory();
    Analyzer analyzer;
    if (random().nextBoolean()) {
        // no payloads
        analyzer = new Analyzer() {

            @Override
            public TokenStreamComponents createComponents(String fieldName) {
                return new TokenStreamComponents(new MockTokenizer(MockTokenizer.WHITESPACE, true));
            }
        };
    } else {
        // fixed length payloads
        final int length = random().nextInt(200);
        analyzer = new Analyzer() {

            @Override
            public TokenStreamComponents createComponents(String fieldName) {
                Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, true);
                return new TokenStreamComponents(tokenizer, new MockFixedLengthPayloadFilter(random(), tokenizer, length));
            }
        };
    }
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(analyzer).setMaxBufferedDocs(10).setReaderPooling(false).setMergePolicy(newLogMergePolicy(10)));
    for (int j = 0; j < 30; j++) {
        TestIndexWriter.addDocWithIndex(writer, j);
    }
    writer.close();
    dir.resetMaxUsedSizeInBytes();
    dir.setTrackDiskUsage(true);
    long startDiskUsage = dir.getMaxUsedSizeInBytes();
    writer = new IndexWriter(dir, newIndexWriterConfig(analyzer).setOpenMode(OpenMode.APPEND).setMaxBufferedDocs(10).setMergeScheduler(new SerialMergeScheduler()).setReaderPooling(false).setMergePolicy(newLogMergePolicy(10)));
    for (int j = 0; j < 1470; j++) {
        TestIndexWriter.addDocWithIndex(writer, j);
    }
    long midDiskUsage = dir.getMaxUsedSizeInBytes();
    dir.resetMaxUsedSizeInBytes();
    writer.forceMerge(1);
    writer.close();
    DirectoryReader.open(dir).close();
    long endDiskUsage = dir.getMaxUsedSizeInBytes();
    // Ending index is 50X as large as starting index; due
    // to 3X disk usage normally we allow 150X max
    // transient usage.  If something is wrong w/ deleter
    // and it doesn't delete intermediate segments then it
    // will exceed this 150X:
    // System.out.println("start " + startDiskUsage + "; mid " + midDiskUsage + ";end " + endDiskUsage);
    assertTrue("writer used too much space while adding documents: mid=" + midDiskUsage + " start=" + startDiskUsage + " end=" + endDiskUsage + " max=" + (startDiskUsage * 150), midDiskUsage < 150 * startDiskUsage);
    assertTrue("writer used too much space after close: endDiskUsage=" + endDiskUsage + " startDiskUsage=" + startDiskUsage + " max=" + (startDiskUsage * 150), endDiskUsage < 150 * startDiskUsage);
    dir.close();
}

Example 7 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class TestIndexWriter method testDeleteAllNRTLeftoverFiles.

public void testDeleteAllNRTLeftoverFiles() throws Exception {
    MockDirectoryWrapper d = new MockDirectoryWrapper(random(), new RAMDirectory());
    IndexWriter w = new IndexWriter(d, new IndexWriterConfig(new MockAnalyzer(random())));
    Document doc = new Document();
    for (int i = 0; i < 20; i++) {
        for (int j = 0; j < 100; ++j) {
            w.addDocument(doc);
        }
        w.commit();
        DirectoryReader.open(w).close();
        w.deleteAll();
        w.commit();
        // Make sure we accumulate no files except for empty
        // segments_N and segments.gen:
        assertTrue(d.listAll().length <= 2);
    }
    w.close();
    d.close();
}

Example 8 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class TestIndexWriterDelete method testErrorInDocsWriterAdd.

// This test tests that the files created by the docs writer before
// a segment is written are cleaned up if there's an i/o error
public void testErrorInDocsWriterAdd() throws IOException {
    MockDirectoryWrapper.Failure failure = new MockDirectoryWrapper.Failure() {

        boolean failed = false;

        @Override
        public MockDirectoryWrapper.Failure reset() {
            failed = false;
            return this;
        }

        @Override
        public void eval(MockDirectoryWrapper dir) throws IOException {
            if (!failed) {
                failed = true;
                throw new IOException("fail in add doc");
            }
        }
    };
    // create a couple of files
    String[] keywords = { "1", "2" };
    String[] unindexed = { "Netherlands", "Italy" };
    String[] unstored = { "Amsterdam has lots of bridges", "Venice has lots of canals" };
    String[] text = { "Amsterdam", "Venice" };
    MockDirectoryWrapper dir = newMockDirectory();
    IndexWriter modifier = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random(), MockTokenizer.WHITESPACE, false)));
    modifier.commit();
    dir.failOn(failure.reset());
    FieldType custom1 = new FieldType();
    custom1.setStored(true);
    for (int i = 0; i < keywords.length; i++) {
        Document doc = new Document();
        doc.add(newStringField("id", keywords[i], Field.Store.YES));
        doc.add(newField("country", unindexed[i], custom1));
        doc.add(newTextField("contents", unstored[i], Field.Store.NO));
        doc.add(newTextField("city", text[i], Field.Store.YES));
        try {
            modifier.addDocument(doc);
        } catch (IOException io) {
            if (VERBOSE) {
                System.out.println("TEST: got expected exc:");
                io.printStackTrace(System.out);
            }
            break;
        }
    }
    assertTrue(modifier.deleter.isClosed());
    TestIndexWriter.assertNoUnreferencedFiles(dir, "docsWriter.abort() failed to delete unreferenced files");
    dir.close();
}

Example 9 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class TestDemoParallelLeafReader method getReindexerNewDVFields.

/** Schema change by adding a new number_<schemaGen> DV field each time. */
private ReindexingReader getReindexerNewDVFields(Path root, final AtomicLong currentSchemaGen) throws IOException {
    return new ReindexingReader(root) {

        @Override
        protected IndexWriterConfig getIndexWriterConfig() throws IOException {
            IndexWriterConfig iwc = newIndexWriterConfig();
            TieredMergePolicy tmp = new TieredMergePolicy();
            // We write tiny docs, so we need tiny floor to avoid O(N^2) merging:
            tmp.setFloorSegmentMB(.01);
            iwc.setMergePolicy(tmp);
            return iwc;
        }

        @Override
        protected Directory openDirectory(Path path) throws IOException {
            MockDirectoryWrapper dir = newMockFSDirectory(path);
            dir.setUseSlowOpenClosers(false);
            dir.setThrottling(Throttling.NEVER);
            return dir;
        }

        @Override
        protected void reindex(long oldSchemaGen, long newSchemaGen, LeafReader reader, Directory parallelDir) throws IOException {
            IndexWriterConfig iwc = newIndexWriterConfig();
            // The order of our docIDs must precisely matching incoming reader:
            iwc.setMergePolicy(new LogByteSizeMergePolicy());
            IndexWriter w = new IndexWriter(parallelDir, iwc);
            int maxDoc = reader.maxDoc();
            if (oldSchemaGen <= 0) {
                // Must slowly parse the stored field into a new doc values field:
                for (int i = 0; i < maxDoc; i++) {
                    // TODO: is this still O(blockSize^2)?
                    Document oldDoc = reader.document(i);
                    Document newDoc = new Document();
                    long value = Long.parseLong(oldDoc.get("text").split(" ")[1]);
                    newDoc.add(new NumericDocValuesField("number_" + newSchemaGen, value));
                    newDoc.add(new LongPoint("number", value));
                    w.addDocument(newDoc);
                }
            } else {
                // Just carry over doc values from previous field:
                NumericDocValues oldValues = reader.getNumericDocValues("number_" + oldSchemaGen);
                assertNotNull("oldSchemaGen=" + oldSchemaGen, oldValues);
                for (int i = 0; i < maxDoc; i++) {
                    // TODO: is this still O(blockSize^2)?
                    assertEquals(i, oldValues.nextDoc());
                    Document oldDoc = reader.document(i);
                    Document newDoc = new Document();
                    newDoc.add(new NumericDocValuesField("number_" + newSchemaGen, oldValues.longValue()));
                    w.addDocument(newDoc);
                }
            }
            w.forceMerge(1);
            w.close();
        }

        @Override
        protected long getCurrentSchemaGen() {
            return currentSchemaGen.get();
        }

        @Override
        protected void checkParallelReader(LeafReader r, LeafReader parR, long schemaGen) throws IOException {
            String fieldName = "number_" + schemaGen;
            if (DEBUG)
                System.out.println(Thread.currentThread().getName() + ": TEST: now check parallel number DVs field=" + fieldName + " r=" + r + " parR=" + parR);
            NumericDocValues numbers = parR.getNumericDocValues(fieldName);
            if (numbers == null) {
                return;
            }
            int maxDoc = r.maxDoc();
            boolean failed = false;
            for (int i = 0; i < maxDoc; i++) {
                Document oldDoc = r.document(i);
                long value = Long.parseLong(oldDoc.get("text").split(" ")[1]);
                assertEquals(i, numbers.nextDoc());
                if (value != numbers.longValue()) {
                    if (DEBUG)
                        System.out.println("FAIL: docID=" + i + " " + oldDoc + " value=" + value + " number=" + numbers.longValue() + " numbers=" + numbers);
                    failed = true;
                } else if (failed) {
                    if (DEBUG)
                        System.out.println("OK: docID=" + i + " " + oldDoc + " value=" + value + " number=" + numbers.longValue());
                }
            }
            assertFalse("FAILED field=" + fieldName + " r=" + r, failed);
        }
    };
}

Example 10 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class TestConcurrentMergeScheduler method testFlushExceptions.

// Make sure running BG merges still work fine even when
// we are hitting exceptions during flushing.
public void testFlushExceptions() throws IOException {
    MockDirectoryWrapper directory = newMockDirectory();
    FailOnlyOnFlush failure = new FailOnlyOnFlush();
    directory.failOn(failure);
    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(2);
    if (iwc.getMergeScheduler() instanceof ConcurrentMergeScheduler) {
        iwc.setMergeScheduler(new SuppressingConcurrentMergeScheduler() {

            @Override
            protected boolean isOK(Throwable th) {
                return th instanceof AlreadyClosedException || (th instanceof IllegalStateException && th.getMessage().contains("this writer hit an unrecoverable error"));
            }
        });
    }
    IndexWriter writer = new IndexWriter(directory, iwc);
    Document doc = new Document();
    Field idField = newStringField("id", "", Field.Store.YES);
    doc.add(idField);
    outer: for (int i = 0; i < 10; i++) {
        if (VERBOSE) {
            System.out.println("TEST: iter=" + i);
        }
        for (int j = 0; j < 20; j++) {
            idField.setStringValue(Integer.toString(i * 20 + j));
            writer.addDocument(doc);
        }
        // flush, and we don't hit the exception
        while (true) {
            writer.addDocument(doc);
            failure.setDoFail();
            try {
                writer.flush(true, true);
                if (failure.hitExc) {
                    fail("failed to hit IOException");
                }
            } catch (IOException ioe) {
                if (VERBOSE) {
                    ioe.printStackTrace(System.out);
                }
                failure.clearDoFail();
                assertTrue(writer.isClosed());
                // Abort should have closed the deleter:
                assertTrue(writer.deleter.isClosed());
                break outer;
            }
        }
    }
    assertFalse(DirectoryReader.indexExists(directory));
    directory.close();
}