Examples with MockDirectoryWrapper org.apache.lucene.store.MockDirectoryWrapper used on opensource projects

Example 51 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class TestBKD method testWithExceptions.

/** Make sure we close open files, delete temp files, etc., on exception */
public void testWithExceptions() throws Exception {
    int numDocs = atLeast(10000);
    int numBytesPerDim = TestUtil.nextInt(random(), 2, 30);
    int numDims = TestUtil.nextInt(random(), 1, 5);
    byte[][][] docValues = new byte[numDocs][][];
    for (int docID = 0; docID < numDocs; docID++) {
        byte[][] values = new byte[numDims][];
        for (int dim = 0; dim < numDims; dim++) {
            values[dim] = new byte[numBytesPerDim];
            random().nextBytes(values[dim]);
        }
        docValues[docID] = values;
    }
    double maxMBHeap = 0.05;
    // Keep retrying until we 1) we allow a big enough heap, and 2) we hit a random IOExc from MDW:
    boolean done = false;
    while (done == false) {
        MockDirectoryWrapper dir = newMockFSDirectory(createTempDir());
        try {
            dir.setRandomIOExceptionRate(0.05);
            dir.setRandomIOExceptionRateOnOpen(0.05);
            verify(dir, docValues, null, numDims, numBytesPerDim, 50, maxMBHeap);
        } catch (IllegalArgumentException iae) {
            // This just means we got a too-small maxMB for the maxPointsInLeafNode; just retry w/ more heap
            assertTrue(iae.getMessage().contains("either increase maxMBSortInHeap or decrease maxPointsInLeafNode"));
            maxMBHeap *= 1.25;
        } catch (IOException ioe) {
            if (ioe.getMessage().contains("a random IOException")) {
                // BKDWriter should fully clean up after itself:
                done = true;
            } else {
                throw ioe;
            }
        }
        String[] files = dir.listAll();
        assertTrue("files=" + Arrays.toString(files), files.length == 0 || Arrays.equals(files, new String[] { "extra0" }));
        dir.close();
    }
}

Example 52 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class PayloadHelper method setUp.

/**
   * Sets up a RAMDirectory, and adds documents (using English.intToEnglish()) with two fields: field and multiField
   * and analyzes them using the PayloadAnalyzer
   * @param similarity The Similarity class to use in the Searcher
   * @param numDocs The num docs to add
   * @return An IndexSearcher
   */
// TODO: randomize
public IndexSearcher setUp(Random random, Similarity similarity, int numDocs) throws IOException {
    Directory directory = new MockDirectoryWrapper(random, new RAMDirectory());
    PayloadAnalyzer analyzer = new PayloadAnalyzer();
    // TODO randomize this
    IndexWriter writer = new IndexWriter(directory, new IndexWriterConfig(analyzer).setSimilarity(similarity));
    // writer.infoStream = System.out;
    for (int i = 0; i < numDocs; i++) {
        Document doc = new Document();
        doc.add(new TextField(FIELD, English.intToEnglish(i), Field.Store.YES));
        doc.add(new TextField(MULTI_FIELD, English.intToEnglish(i) + "  " + English.intToEnglish(i), Field.Store.YES));
        doc.add(new TextField(NO_PAYLOAD_FIELD, English.intToEnglish(i), Field.Store.YES));
        writer.addDocument(doc);
    }
    writer.forceMerge(1);
    reader = DirectoryReader.open(writer);
    writer.close();
    IndexSearcher searcher = LuceneTestCase.newSearcher(LuceneTestCase.getOnlyLeafReader(reader));
    searcher.setSimilarity(similarity);
    return searcher;
}

Example 53 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class IndexAndTaxonomyReplicationClientTest method testConsistencyOnExceptions.

/*
   * This test verifies that the client and handler do not end up in a corrupt
   * index if exceptions are thrown at any point during replication. Either when
   * a client copies files from the server to the temporary space, or when the
   * handler copies them to the index directory.
   */
@Test
public void testConsistencyOnExceptions() throws Exception {
    // so the handler's index isn't empty
    replicator.publish(createRevision(1));
    client.updateNow();
    client.close();
    callback.close();
    // wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors
    final SourceDirectoryFactory in = sourceDirFactory;
    final AtomicInteger failures = new AtomicInteger(atLeast(10));
    sourceDirFactory = new SourceDirectoryFactory() {

        private long clientMaxSize = 100, handlerIndexMaxSize = 100, handlerTaxoMaxSize = 100;

        private double clientExRate = 1.0, handlerIndexExRate = 1.0, handlerTaxoExRate = 1.0;

        @Override
        public void cleanupSession(String sessionID) throws IOException {
            in.cleanupSession(sessionID);
        }

        @SuppressWarnings("synthetic-access")
        @Override
        public Directory getDirectory(String sessionID, String source) throws IOException {
            Directory dir = in.getDirectory(sessionID, source);
            if (random().nextBoolean() && failures.get() > 0) {
                // client should fail, return wrapped dir
                MockDirectoryWrapper mdw = new MockDirectoryWrapper(random(), dir);
                mdw.setRandomIOExceptionRateOnOpen(clientExRate);
                mdw.setMaxSizeInBytes(clientMaxSize);
                mdw.setRandomIOExceptionRate(clientExRate);
                mdw.setCheckIndexOnClose(false);
                clientMaxSize *= 2;
                clientExRate /= 2;
                return mdw;
            }
            if (failures.get() > 0 && random().nextBoolean()) {
                // handler should fail
                if (random().nextBoolean()) {
                    // index dir fail
                    handlerIndexDir.setMaxSizeInBytes(handlerIndexMaxSize);
                    handlerIndexDir.setRandomIOExceptionRate(handlerIndexExRate);
                    handlerIndexDir.setRandomIOExceptionRateOnOpen(handlerIndexExRate);
                    handlerIndexMaxSize *= 2;
                    handlerIndexExRate /= 2;
                } else {
                    // taxo dir fail
                    handlerTaxoDir.setMaxSizeInBytes(handlerTaxoMaxSize);
                    handlerTaxoDir.setRandomIOExceptionRate(handlerTaxoExRate);
                    handlerTaxoDir.setRandomIOExceptionRateOnOpen(handlerTaxoExRate);
                    handlerTaxoDir.setCheckIndexOnClose(false);
                    handlerTaxoMaxSize *= 2;
                    handlerTaxoExRate /= 2;
                }
            } else {
                // disable all errors
                handlerIndexDir.setMaxSizeInBytes(0);
                handlerIndexDir.setRandomIOExceptionRate(0.0);
                handlerIndexDir.setRandomIOExceptionRateOnOpen(0.0);
                handlerTaxoDir.setMaxSizeInBytes(0);
                handlerTaxoDir.setRandomIOExceptionRate(0.0);
                handlerTaxoDir.setRandomIOExceptionRateOnOpen(0.0);
            }
            return dir;
        }
    };
    handler = new IndexAndTaxonomyReplicationHandler(handlerIndexDir, handlerTaxoDir, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            if (random().nextDouble() < 0.2 && failures.get() > 0) {
                throw new RuntimeException("random exception from callback");
            }
            return null;
        }
    });
    final AtomicBoolean failed = new AtomicBoolean();
    // wrap handleUpdateException so we can act on the thrown exception
    client = new ReplicationClient(replicator, handler, sourceDirFactory) {

        @SuppressWarnings("synthetic-access")
        @Override
        protected void handleUpdateException(Throwable t) {
            if (t instanceof IOException) {
                try {
                    if (VERBOSE) {
                        System.out.println("hit exception during update: " + t);
                        t.printStackTrace(System.out);
                    }
                    // test that the index can be read and also some basic statistics
                    DirectoryReader reader = DirectoryReader.open(handlerIndexDir.getDelegate());
                    try {
                        int numDocs = reader.numDocs();
                        int version = Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
                        assertEquals(numDocs, version);
                    } finally {
                        reader.close();
                    }
                    // verify index is fully consistent
                    TestUtil.checkIndex(handlerIndexDir.getDelegate());
                    // verify taxonomy index is fully consistent (since we only add one
                    // category to all documents, there's nothing much more to validate.
                    ByteArrayOutputStream bos = new ByteArrayOutputStream(1024);
                    CheckIndex.Status indexStatus = null;
                    try (CheckIndex checker = new CheckIndex(handlerTaxoDir.getDelegate())) {
                        checker.setFailFast(true);
                        checker.setInfoStream(new PrintStream(bos, false, IOUtils.UTF_8), false);
                        try {
                            indexStatus = checker.checkIndex(null);
                        } catch (IOException | RuntimeException ioe) {
                        // ok: we fallback below
                        }
                    }
                } catch (IOException e) {
                    failed.set(true);
                    throw new RuntimeException(e);
                } catch (RuntimeException e) {
                    failed.set(true);
                    throw e;
                } finally {
                    // count-down number of failures
                    failures.decrementAndGet();
                    assert failures.get() >= 0 : "handler failed too many times: " + failures.get();
                    if (VERBOSE) {
                        if (failures.get() == 0) {
                            System.out.println("no more failures expected");
                        } else {
                            System.out.println("num failures left: " + failures.get());
                        }
                    }
                }
            } else {
                failed.set(true);
                if (t instanceof RuntimeException) {
                    throw (RuntimeException) t;
                }
                throw new RuntimeException(t);
            }
        }
    };
    client.startUpdateThread(10, "indexAndTaxo");
    final Directory baseHandlerIndexDir = handlerIndexDir.getDelegate();
    int numRevisions = atLeast(20) + 2;
    for (int i = 2; i < numRevisions && failed.get() == false; i++) {
        replicator.publish(createRevision(i));
        assertHandlerRevision(i, baseHandlerIndexDir);
    }
    // disable errors -- maybe randomness didn't exhaust all allowed failures,
    // and we don't want e.g. CheckIndex to hit false errors. 
    handlerIndexDir.setMaxSizeInBytes(0);
    handlerIndexDir.setRandomIOExceptionRate(0.0);
    handlerIndexDir.setRandomIOExceptionRateOnOpen(0.0);
    handlerTaxoDir.setMaxSizeInBytes(0);
    handlerTaxoDir.setRandomIOExceptionRate(0.0);
    handlerTaxoDir.setRandomIOExceptionRateOnOpen(0.0);
}

Example 54 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class IndexReplicationClientTest method testConsistencyOnExceptions.

/*
   * This test verifies that the client and handler do not end up in a corrupt
   * index if exceptions are thrown at any point during replication. Either when
   * a client copies files from the server to the temporary space, or when the
   * handler copies them to the index directory.
   */
@Test
public void testConsistencyOnExceptions() throws Exception {
    // so the handler's index isn't empty
    replicator.publish(createRevision(1));
    client.updateNow();
    client.close();
    callback.close();
    // wrap sourceDirFactory to return a MockDirWrapper so we can simulate errors
    final SourceDirectoryFactory in = sourceDirFactory;
    final AtomicInteger failures = new AtomicInteger(atLeast(10));
    sourceDirFactory = new SourceDirectoryFactory() {

        private long clientMaxSize = 100, handlerMaxSize = 100;

        private double clientExRate = 1.0, handlerExRate = 1.0;

        @Override
        public void cleanupSession(String sessionID) throws IOException {
            in.cleanupSession(sessionID);
        }

        @SuppressWarnings("synthetic-access")
        @Override
        public Directory getDirectory(String sessionID, String source) throws IOException {
            Directory dir = in.getDirectory(sessionID, source);
            if (random().nextBoolean() && failures.get() > 0) {
                // client should fail, return wrapped dir
                MockDirectoryWrapper mdw = new MockDirectoryWrapper(random(), dir);
                mdw.setRandomIOExceptionRateOnOpen(clientExRate);
                mdw.setMaxSizeInBytes(clientMaxSize);
                mdw.setRandomIOExceptionRate(clientExRate);
                mdw.setCheckIndexOnClose(false);
                clientMaxSize *= 2;
                clientExRate /= 2;
                return mdw;
            }
            if (failures.get() > 0 && random().nextBoolean()) {
                // handler should fail
                handlerDir.setMaxSizeInBytes(handlerMaxSize);
                handlerDir.setRandomIOExceptionRateOnOpen(handlerExRate);
                handlerDir.setRandomIOExceptionRate(handlerExRate);
                handlerMaxSize *= 2;
                handlerExRate /= 2;
            } else {
                // disable errors
                handlerDir.setMaxSizeInBytes(0);
                handlerDir.setRandomIOExceptionRate(0.0);
                handlerDir.setRandomIOExceptionRateOnOpen(0.0);
            }
            return dir;
        }
    };
    handler = new IndexReplicationHandler(handlerDir, new Callable<Boolean>() {

        @Override
        public Boolean call() throws Exception {
            if (random().nextDouble() < 0.2 && failures.get() > 0) {
                throw new RuntimeException("random exception from callback");
            }
            return null;
        }
    });
    // wrap handleUpdateException so we can act on the thrown exception
    client = new ReplicationClient(replicator, handler, sourceDirFactory) {

        @SuppressWarnings("synthetic-access")
        @Override
        protected void handleUpdateException(Throwable t) {
            if (t instanceof IOException) {
                if (VERBOSE) {
                    System.out.println("hit exception during update: " + t);
                    t.printStackTrace(System.out);
                }
                try {
                    // test that the index can be read and also some basic statistics
                    DirectoryReader reader = DirectoryReader.open(handlerDir.getDelegate());
                    try {
                        int numDocs = reader.numDocs();
                        int version = Integer.parseInt(reader.getIndexCommit().getUserData().get(VERSION_ID), 16);
                        assertEquals(numDocs, version);
                    } finally {
                        reader.close();
                    }
                    // verify index consistency
                    TestUtil.checkIndex(handlerDir.getDelegate());
                } catch (IOException e) {
                    // exceptions here are bad, don't ignore them
                    throw new RuntimeException(e);
                } finally {
                    // count-down number of failures
                    failures.decrementAndGet();
                    assert failures.get() >= 0 : "handler failed too many times: " + failures.get();
                    if (VERBOSE) {
                        if (failures.get() == 0) {
                            System.out.println("no more failures expected");
                        } else {
                            System.out.println("num failures left: " + failures.get());
                        }
                    }
                }
            } else {
                if (t instanceof RuntimeException)
                    throw (RuntimeException) t;
                throw new RuntimeException(t);
            }
        }
    };
    client.startUpdateThread(10, "index");
    final Directory baseHandlerDir = handlerDir.getDelegate();
    int numRevisions = atLeast(20);
    for (int i = 2; i < numRevisions; i++) {
        replicator.publish(createRevision(i));
        assertHandlerRevision(i, baseHandlerDir);
    }
    // disable errors -- maybe randomness didn't exhaust all allowed failures,
    // and we don't want e.g. CheckIndex to hit false errors. 
    handlerDir.setMaxSizeInBytes(0);
    handlerDir.setRandomIOExceptionRate(0.0);
    handlerDir.setRandomIOExceptionRateOnOpen(0.0);
}

Example 55 with MockDirectoryWrapper

use of org.apache.lucene.store.MockDirectoryWrapper in project lucene-solr by apache.

the class LuceneTestCase method wrapDirectory.

private static BaseDirectoryWrapper wrapDirectory(Random random, Directory directory, boolean bare) {
    if (rarely(random) && !bare) {
        directory = new NRTCachingDirectory(directory, random.nextDouble(), random.nextDouble());
    }
    if (bare) {
        BaseDirectoryWrapper base = new RawDirectoryWrapper(directory);
        closeAfterSuite(new CloseableDirectory(base, suiteFailureMarker));
        return base;
    } else {
        MockDirectoryWrapper mock = new MockDirectoryWrapper(random, directory);
        mock.setThrottling(TEST_THROTTLING);
        closeAfterSuite(new CloseableDirectory(mock, suiteFailureMarker));
        return mock;
    }
}