Examples with BaseDirectoryWrapper org.apache.lucene.store.BaseDirectoryWrapper used on opensource projects

Example 11 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class Test2BDocs method test2BDocs.

// indexes Integer.MAX_VALUE docs with indexed field(s)
public void test2BDocs() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BDocs"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setRAMBufferSizeMB(256.0).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy(false, 10)).setOpenMode(IndexWriterConfig.OpenMode.CREATE).setCodec(TestUtil.getDefaultCodec()));
    Document doc = new Document();
    Field field = new Field("f1", "a", StringField.TYPE_NOT_STORED);
    doc.add(field);
    for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
        w.addDocument(doc);
        if (i % (10 * 1000 * 1000) == 0) {
            System.out.println("indexed: " + i);
            System.out.flush();
        }
    }
    w.forceMerge(1);
    w.close();
    System.out.println("verifying...");
    System.out.flush();
    DirectoryReader r = DirectoryReader.open(dir);
    BytesRef term = new BytesRef(1);
    term.bytes[0] = (byte) 'a';
    term.length = 1;
    long skips = 0;
    Random rnd = random();
    long start = System.nanoTime();
    for (LeafReaderContext context : r.leaves()) {
        LeafReader reader = context.reader();
        int lim = context.reader().maxDoc();
        Terms terms = reader.fields().terms("f1");
        for (int i = 0; i < 10000; i++) {
            TermsEnum te = terms.iterator();
            assertTrue(te.seekExact(term));
            PostingsEnum docs = te.postings(null);
            // skip randomly through the term
            for (int target = -1; ; ) {
                int maxSkipSize = lim - target + 1;
                // do a smaller skip half of the time
                if (rnd.nextBoolean()) {
                    maxSkipSize = Math.min(256, maxSkipSize);
                }
                int newTarget = target + rnd.nextInt(maxSkipSize) + 1;
                if (newTarget >= lim) {
                    // we already skipped to end, so break.
                    if (target + 1 >= lim)
                        break;
                    // skip to end
                    newTarget = lim - 1;
                }
                target = newTarget;
                int res = docs.advance(target);
                if (res == PostingsEnum.NO_MORE_DOCS)
                    break;
                assertTrue(res >= target);
                skips++;
                target = res;
            }
        }
    }
    r.close();
    dir.close();
    long end = System.nanoTime();
    System.out.println("Skip count=" + skips + " seconds=" + TimeUnit.NANOSECONDS.toSeconds(end - start));
    assert skips > 0;
}

Example 12 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class Test2BNumericDocValues method testNumerics.

// indexes IndexWriter.MAX_DOCS docs with an increasing dv field
public void testNumerics() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BNumerics"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setRAMBufferSizeMB(256.0).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy(false, 10)).setOpenMode(IndexWriterConfig.OpenMode.CREATE).setCodec(TestUtil.getDefaultCodec()));
    Document doc = new Document();
    NumericDocValuesField dvField = new NumericDocValuesField("dv", 0);
    doc.add(dvField);
    for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
        dvField.setLongValue(i);
        w.addDocument(doc);
        if (i % 100000 == 0) {
            System.out.println("indexed: " + i);
            System.out.flush();
        }
    }
    w.forceMerge(1);
    w.close();
    System.out.println("verifying...");
    System.out.flush();
    DirectoryReader r = DirectoryReader.open(dir);
    long expectedValue = 0;
    for (LeafReaderContext context : r.leaves()) {
        LeafReader reader = context.reader();
        NumericDocValues dv = reader.getNumericDocValues("dv");
        for (int i = 0; i < reader.maxDoc(); i++) {
            assertEquals(i, dv.nextDoc());
            assertEquals(expectedValue, dv.longValue());
            expectedValue++;
        }
    }
    r.close();
    dir.close();
}

Example 13 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class Test2BPositions method test.

public void test() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BPositions"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setRAMBufferSizeMB(256.0).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy(false, 10)).setOpenMode(IndexWriterConfig.OpenMode.CREATE).setCodec(TestUtil.getDefaultCodec()));
    MergePolicy mp = w.getConfig().getMergePolicy();
    if (mp instanceof LogByteSizeMergePolicy) {
        // 1 petabyte:
        ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024 * 1024 * 1024);
    }
    Document doc = new Document();
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setOmitNorms(true);
    Field field = new Field("field", new MyTokenStream(), ft);
    doc.add(field);
    final int numDocs = (Integer.MAX_VALUE / 26) + 1;
    for (int i = 0; i < numDocs; i++) {
        w.addDocument(doc);
        if (VERBOSE && i % 100000 == 0) {
            System.out.println(i + " of " + numDocs + "...");
        }
    }
    w.forceMerge(1);
    w.close();
    dir.close();
}

Example 14 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class TestIndexWriterExceptions method testSegmentsChecksumError.

// LUCENE-1044: Simulate checksum error in segments_N
public void testSegmentsChecksumError() throws IOException {
    BaseDirectoryWrapper dir = newDirectory();
    // we corrupt the index
    dir.setCheckIndexOnClose(false);
    IndexWriter writer = null;
    writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())));
    // add 100 documents
    for (int i = 0; i < 100; i++) {
        addDoc(writer);
    }
    // close
    writer.close();
    long gen = SegmentInfos.getLastCommitGeneration(dir);
    assertTrue("segment generation should be > 0 but got " + gen, gen > 0);
    final String segmentsFileName = SegmentInfos.getLastCommitSegmentsFileName(dir);
    IndexInput in = dir.openInput(segmentsFileName, newIOContext(random()));
    IndexOutput out = dir.createOutput(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", 1 + gen), newIOContext(random()));
    out.copyBytes(in, in.length() - 1);
    byte b = in.readByte();
    out.writeByte((byte) (1 + b));
    out.close();
    in.close();
    expectThrows(CorruptIndexException.class, () -> {
        DirectoryReader.open(dir);
    });
    dir.close();
}

Example 15 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class TestIndexWriterOnJRECrash method checkIndexes.

/**
   * Recursively looks for indexes underneath <code>file</code>,
   * and runs checkindex on them. returns true if it found any indexes.
   */
public boolean checkIndexes(Path path) throws IOException {
    final AtomicBoolean found = new AtomicBoolean();
    Files.walkFileTree(path, new SimpleFileVisitor<Path>() {

        @Override
        public FileVisitResult postVisitDirectory(Path dirPath, IOException exc) throws IOException {
            if (exc != null) {
                throw exc;
            } else {
                try (BaseDirectoryWrapper dir = newFSDirectory(dirPath)) {
                    // don't double-checkindex
                    dir.setCheckIndexOnClose(false);
                    if (DirectoryReader.indexExists(dir)) {
                        if (VERBOSE) {
                            System.err.println("Checking index: " + dirPath);
                        }
                        // since that too risky):
                        if (SegmentInfos.getLastCommitGeneration(dir) > 1) {
                            TestUtil.checkIndex(dir);
                        }
                        found.set(true);
                    }
                }
                return FileVisitResult.CONTINUE;
            }
        }
    });
    return found.get();
}