Examples with MockDirectoryWrapper org.apache.lucene.store.MockDirectoryWrapper used on opensource projects

Example 61 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project elasticsearch by elastic.

the class ShadowEngineTests method testFailStart.

/**
     * Random test that throws random exception and ensures all references are
     * counted down / released and resources are closed.
     */
public void testFailStart() throws IOException {
    // Need a commit point for this
    ParsedDocument doc = testParsedDocument("1", "test", null, testDocumentWithTextField(), B_1, null);
    primaryEngine.index(indexForDoc(doc));
    primaryEngine.flush();
    // this test fails if any reader, searcher or directory is not closed - MDW FTW
    final int iters = scaledRandomIntBetween(10, 100);
    for (int i = 0; i < iters; i++) {
        MockDirectoryWrapper wrapper = newMockFSDirectory(dirPath);
        wrapper.setFailOnOpenInput(randomBoolean());
        wrapper.setAllowRandomFileNotFoundException(randomBoolean());
        wrapper.setRandomIOExceptionRate(randomDouble());
        wrapper.setRandomIOExceptionRateOnOpen(randomDouble());
        try (Store store = createStore(wrapper)) {
            int refCount = store.refCount();
            assertTrue("refCount: " + store.refCount(), store.refCount() > 0);
            ShadowEngine holder;
            try {
                holder = createShadowEngine(store);
            } catch (EngineCreationFailureException ex) {
                assertEquals(store.refCount(), refCount);
                continue;
            }
            assertEquals(store.refCount(), refCount + 1);
            final int numStarts = scaledRandomIntBetween(1, 5);
            for (int j = 0; j < numStarts; j++) {
                try {
                    assertEquals(store.refCount(), refCount + 1);
                    holder.close();
                    holder = createShadowEngine(store);
                    assertEquals(store.refCount(), refCount + 1);
                } catch (EngineCreationFailureException ex) {
                    // all is fine
                    assertEquals(store.refCount(), refCount);
                    break;
                }
            }
            holder.close();
            assertEquals(store.refCount(), refCount);
        }
    }
}

Example 62 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project elasticsearch by elastic.

the class InternalEngineTests method testFailEngineOnRandomIO.

public void testFailEngineOnRandomIO() throws IOException, InterruptedException {
    MockDirectoryWrapper wrapper = newMockDirectory();
    final Path translogPath = createTempDir("testFailEngineOnRandomIO");
    try (Store store = createStore(wrapper)) {
        CyclicBarrier join = new CyclicBarrier(2);
        CountDownLatch start = new CountDownLatch(1);
        AtomicInteger controller = new AtomicInteger(0);
        EngineConfig config = config(defaultSettings, store, translogPath, newMergePolicy(), IndexRequest.UNSET_AUTO_GENERATED_TIMESTAMP, new ReferenceManager.RefreshListener() {

            @Override
            public void beforeRefresh() throws IOException {
            }

            @Override
            public void afterRefresh(boolean didRefresh) throws IOException {
                int i = controller.incrementAndGet();
                if (i == 1) {
                    throw new MockDirectoryWrapper.FakeIOException();
                } else if (i == 2) {
                    try {
                        start.await();
                    } catch (InterruptedException e) {
                        throw new AssertionError(e);
                    }
                    throw new ElasticsearchException("something completely different");
                }
            }
        });
        InternalEngine internalEngine = new InternalEngine(config);
        int docId = 0;
        final ParsedDocument doc = testParsedDocument(Integer.toString(docId), "test", null, testDocumentWithTextField(), new BytesArray("{}".getBytes(Charset.defaultCharset())), null);
        Engine.Index index = randomBoolean() ? indexForDoc(doc) : randomAppendOnly(doc, false, docId);
        internalEngine.index(index);
        Runnable r = () -> {
            try {
                join.await();
            } catch (Exception e) {
                throw new AssertionError(e);
            }
            try {
                internalEngine.refresh("test");
                fail();
            } catch (AlreadyClosedException ex) {
                if (ex.getCause() != null) {
                    assertTrue(ex.toString(), ex.getCause() instanceof MockDirectoryWrapper.FakeIOException);
                }
            } catch (RefreshFailedEngineException ex) {
            // fine
            } finally {
                start.countDown();
            }
        };
        Thread t = new Thread(r);
        Thread t1 = new Thread(r);
        t.start();
        t1.start();
        t.join();
        t1.join();
        assertTrue(internalEngine.isClosed.get());
        assertTrue(internalEngine.failedEngine.get() instanceof MockDirectoryWrapper.FakeIOException);
    }
}

Example 63 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class TestDemoParallelLeafReader method getReindexerSameDVField.

/** Schema change by adding changing how the same "number" DV field is indexed. */
private ReindexingReader getReindexerSameDVField(Path root, final AtomicLong currentSchemaGen, final AtomicLong mergingSchemaGen) throws IOException {
    return new ReindexingReader(root) {

        @Override
        protected IndexWriterConfig getIndexWriterConfig() throws IOException {
            IndexWriterConfig iwc = newIndexWriterConfig();
            TieredMergePolicy tmp = new TieredMergePolicy();
            // We write tiny docs, so we need tiny floor to avoid O(N^2) merging:
            tmp.setFloorSegmentMB(.01);
            iwc.setMergePolicy(tmp);
            if (TEST_NIGHTLY) {
                // during nightly tests, we might use too many files if we arent careful
                iwc.setUseCompoundFile(true);
            }
            return iwc;
        }

        @Override
        protected Directory openDirectory(Path path) throws IOException {
            MockDirectoryWrapper dir = newMockFSDirectory(path);
            dir.setUseSlowOpenClosers(false);
            dir.setThrottling(Throttling.NEVER);
            return dir;
        }

        @Override
        protected void reindex(long oldSchemaGen, long newSchemaGen, LeafReader reader, Directory parallelDir) throws IOException {
            IndexWriterConfig iwc = newIndexWriterConfig();
            // The order of our docIDs must precisely matching incoming reader:
            iwc.setMergePolicy(new LogByteSizeMergePolicy());
            IndexWriter w = new IndexWriter(parallelDir, iwc);
            int maxDoc = reader.maxDoc();
            if (oldSchemaGen <= 0) {
                // Must slowly parse the stored field into a new doc values field:
                for (int i = 0; i < maxDoc; i++) {
                    // TODO: is this still O(blockSize^2)?
                    Document oldDoc = reader.document(i);
                    Document newDoc = new Document();
                    long value = Long.parseLong(oldDoc.get("text").split(" ")[1]);
                    newDoc.add(new NumericDocValuesField("number", newSchemaGen * value));
                    newDoc.add(new LongPoint("number", value));
                    w.addDocument(newDoc);
                }
            } else {
                // Just carry over doc values from previous field:
                NumericDocValues oldValues = reader.getNumericDocValues("number");
                assertNotNull("oldSchemaGen=" + oldSchemaGen, oldValues);
                for (int i = 0; i < maxDoc; i++) {
                    // TODO: is this still O(blockSize^2)?
                    Document oldDoc = reader.document(i);
                    Document newDoc = new Document();
                    assertEquals(i, oldValues.nextDoc());
                    newDoc.add(new NumericDocValuesField("number", newSchemaGen * (oldValues.longValue() / oldSchemaGen)));
                    w.addDocument(newDoc);
                }
            }
            w.forceMerge(1);
            w.close();
        }

        @Override
        protected long getCurrentSchemaGen() {
            return currentSchemaGen.get();
        }

        @Override
        protected long getMergingSchemaGen() {
            return mergingSchemaGen.get();
        }

        @Override
        protected void checkParallelReader(LeafReader r, LeafReader parR, long schemaGen) throws IOException {
            if (DEBUG)
                System.out.println(Thread.currentThread().getName() + ": TEST: now check parallel number DVs r=" + r + " parR=" + parR);
            NumericDocValues numbers = parR.getNumericDocValues("numbers");
            if (numbers == null) {
                return;
            }
            int maxDoc = r.maxDoc();
            boolean failed = false;
            for (int i = 0; i < maxDoc; i++) {
                Document oldDoc = r.document(i);
                long value = Long.parseLong(oldDoc.get("text").split(" ")[1]);
                value *= schemaGen;
                assertEquals(i, numbers.nextDoc());
                if (value != numbers.longValue()) {
                    System.out.println("FAIL: docID=" + i + " " + oldDoc + " value=" + value + " number=" + numbers.longValue() + " numbers=" + numbers);
                    failed = true;
                } else if (failed) {
                    System.out.println("OK: docID=" + i + " " + oldDoc + " value=" + value + " number=" + numbers.longValue());
                }
            }
            assertFalse("FAILED r=" + r, failed);
        }
    };
}

Example 64 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class TestConcurrentMergeScheduler method testNoStallMergeThreads.

// LUCENE-6197
public void testNoStallMergeThreads() throws Exception {
    MockDirectoryWrapper dir = newMockDirectory();
    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    iwc.setMaxBufferedDocs(2);
    IndexWriter w = new IndexWriter(dir, iwc);
    for (int i = 0; i < 1000; i++) {
        Document doc = new Document();
        doc.add(newStringField("field", "" + i, Field.Store.YES));
        w.addDocument(doc);
    }
    w.close();
    iwc = newIndexWriterConfig(new MockAnalyzer(random()));
    AtomicBoolean failed = new AtomicBoolean();
    ConcurrentMergeScheduler cms = new ConcurrentMergeScheduler() {

        @Override
        protected void doStall() {
            if (Thread.currentThread().getName().startsWith("Lucene Merge Thread")) {
                failed.set(true);
            }
            super.doStall();
        }
    };
    cms.setMaxMergesAndThreads(2, 1);
    iwc.setMergeScheduler(cms);
    iwc.setMaxBufferedDocs(2);
    w = new IndexWriter(dir, iwc);
    w.forceMerge(1);
    w.close();
    dir.close();
    assertFalse(failed.get());
}

Example 65 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class TestConcurrentMergeScheduler method testTotalBytesSize.

public void testTotalBytesSize() throws Exception {
    Directory d = newDirectory();
    if (d instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) d).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriterConfig iwc = newIndexWriterConfig(new MockAnalyzer(random()));
    iwc.setMaxBufferedDocs(5);
    CountDownLatch atLeastOneMerge = new CountDownLatch(1);
    iwc.setMergeScheduler(new TrackingCMS(atLeastOneMerge));
    if (TestUtil.getPostingsFormat("id").equals("SimpleText")) {
        // no
        iwc.setCodec(TestUtil.alwaysPostingsFormat(TestUtil.getDefaultPostingsFormat()));
    }
    IndexWriter w = new IndexWriter(d, iwc);
    for (int i = 0; i < 1000; i++) {
        Document doc = new Document();
        doc.add(new StringField("id", "" + i, Field.Store.NO));
        w.addDocument(doc);
        if (random().nextBoolean()) {
            w.deleteDocuments(new Term("id", "" + random().nextInt(i + 1)));
        }
    }
    atLeastOneMerge.await();
    assertTrue(((TrackingCMS) w.getConfig().getMergeScheduler()).totMergedBytes != 0);
    w.close();
    d.close();
}