Examples with BaseDirectoryWrapper org.apache.lucene.store.BaseDirectoryWrapper used on opensource projects

Example 6 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class TestNeverDelete method testIndexing.

public void testIndexing() throws Exception {
    final Path tmpDir = createTempDir("TestNeverDelete");
    final BaseDirectoryWrapper d = newFSDirectory(tmpDir);
    final RandomIndexWriter w = new RandomIndexWriter(random(), d, newIndexWriterConfig(new MockAnalyzer(random())).setIndexDeletionPolicy(NoDeletionPolicy.INSTANCE));
    w.w.getConfig().setMaxBufferedDocs(TestUtil.nextInt(random(), 5, 30));
    w.commit();
    Thread[] indexThreads = new Thread[random().nextInt(4)];
    final long stopTime = System.currentTimeMillis() + atLeast(1000);
    for (int x = 0; x < indexThreads.length; x++) {
        indexThreads[x] = new Thread() {

            @Override
            public void run() {
                try {
                    int docCount = 0;
                    while (System.currentTimeMillis() < stopTime) {
                        final Document doc = new Document();
                        doc.add(newStringField("dc", "" + docCount, Field.Store.YES));
                        doc.add(newTextField("field", "here is some text", Field.Store.YES));
                        w.addDocument(doc);
                        if (docCount % 13 == 0) {
                            w.commit();
                        }
                        docCount++;
                    }
                } catch (Exception e) {
                    throw new RuntimeException(e);
                }
            }
        };
        indexThreads[x].setName("Thread " + x);
        indexThreads[x].start();
    }
    final Set<String> allFiles = new HashSet<>();
    DirectoryReader r = DirectoryReader.open(d);
    while (System.currentTimeMillis() < stopTime) {
        final IndexCommit ic = r.getIndexCommit();
        if (VERBOSE) {
            System.out.println("TEST: check files: " + ic.getFileNames());
        }
        allFiles.addAll(ic.getFileNames());
        // Make sure no old files were removed
        for (String fileName : allFiles) {
            assertTrue("file " + fileName + " does not exist", slowFileExists(d, fileName));
        }
        DirectoryReader r2 = DirectoryReader.openIfChanged(r);
        if (r2 != null) {
            r.close();
            r = r2;
        }
        Thread.sleep(1);
    }
    r.close();
    for (Thread t : indexThreads) {
        t.join();
    }
    w.close();
    d.close();
}

Example 7 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class Test2BPostingsBytes method test.

public void test() throws Exception {
    IndexWriterConfig defaultConfig = new IndexWriterConfig(null);
    Codec defaultCodec = defaultConfig.getCodec();
    if ((new IndexWriterConfig(null)).getCodec() instanceof CompressingCodec) {
        Pattern regex = Pattern.compile("maxDocsPerChunk=(\\d+), blockSize=(\\d+)");
        Matcher matcher = regex.matcher(defaultCodec.toString());
        assertTrue("Unexpected CompressingCodec toString() output: " + defaultCodec.toString(), matcher.find());
        int maxDocsPerChunk = Integer.parseInt(matcher.group(1));
        int blockSize = Integer.parseInt(matcher.group(2));
        int product = maxDocsPerChunk * blockSize;
        assumeTrue(defaultCodec.getName() + " maxDocsPerChunk (" + maxDocsPerChunk + ") * blockSize (" + blockSize + ") < 16 - this can trigger OOM with -Dtests.heapsize=30g", product >= 16);
    }
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BPostingsBytes1"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setRAMBufferSizeMB(256.0).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy(false, 10)).setOpenMode(IndexWriterConfig.OpenMode.CREATE).setCodec(TestUtil.getDefaultCodec()));
    MergePolicy mp = w.getConfig().getMergePolicy();
    if (mp instanceof LogByteSizeMergePolicy) {
        // 1 petabyte:
        ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024 * 1024 * 1024);
    }
    Document doc = new Document();
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setIndexOptions(IndexOptions.DOCS_AND_FREQS);
    ft.setOmitNorms(true);
    MyTokenStream tokenStream = new MyTokenStream();
    Field field = new Field("field", tokenStream, ft);
    doc.add(field);
    final int numDocs = 1000;
    for (int i = 0; i < numDocs; i++) {
        if (i % 2 == 1) {
            // trick blockPF's little optimization
            tokenStream.n = 65536;
        } else {
            tokenStream.n = 65537;
        }
        w.addDocument(doc);
    }
    w.forceMerge(1);
    w.close();
    DirectoryReader oneThousand = DirectoryReader.open(dir);
    DirectoryReader[] subReaders = new DirectoryReader[1000];
    Arrays.fill(subReaders, oneThousand);
    BaseDirectoryWrapper dir2 = newFSDirectory(createTempDir("2BPostingsBytes2"));
    if (dir2 instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir2).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w2 = new IndexWriter(dir2, new IndexWriterConfig(null));
    TestUtil.addIndexesSlowly(w2, subReaders);
    w2.forceMerge(1);
    w2.close();
    oneThousand.close();
    DirectoryReader oneMillion = DirectoryReader.open(dir2);
    subReaders = new DirectoryReader[2000];
    Arrays.fill(subReaders, oneMillion);
    BaseDirectoryWrapper dir3 = newFSDirectory(createTempDir("2BPostingsBytes3"));
    if (dir3 instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir3).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w3 = new IndexWriter(dir3, new IndexWriterConfig(null));
    TestUtil.addIndexesSlowly(w3, subReaders);
    w3.forceMerge(1);
    w3.close();
    oneMillion.close();
    dir.close();
    dir2.close();
    dir3.close();
}

Example 8 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class Test2BTerms method test2BTerms.

public void test2BTerms() throws IOException {
    System.out.println("Starting Test2B");
    final long TERM_COUNT = ((long) Integer.MAX_VALUE) + 100000000;
    final int TERMS_PER_DOC = TestUtil.nextInt(random(), 100000, 1000000);
    List<BytesRef> savedTerms = null;
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BTerms"));
    //MockDirectoryWrapper dir = newFSDirectory(new File("/p/lucene/indices/2bindex"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    // don't double-checkindex
    dir.setCheckIndexOnClose(false);
    if (true) {
        IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setRAMBufferSizeMB(256.0).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy(false, 10)).setOpenMode(IndexWriterConfig.OpenMode.CREATE).setCodec(TestUtil.getDefaultCodec()));
        MergePolicy mp = w.getConfig().getMergePolicy();
        if (mp instanceof LogByteSizeMergePolicy) {
            // 1 petabyte:
            ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024 * 1024 * 1024);
        }
        Document doc = new Document();
        final MyTokenStream ts = new MyTokenStream(random(), TERMS_PER_DOC);
        FieldType customType = new FieldType(TextField.TYPE_NOT_STORED);
        customType.setIndexOptions(IndexOptions.DOCS);
        customType.setOmitNorms(true);
        Field field = new Field("field", ts, customType);
        doc.add(field);
        //w.setInfoStream(System.out);
        final int numDocs = (int) (TERM_COUNT / TERMS_PER_DOC);
        System.out.println("TERMS_PER_DOC=" + TERMS_PER_DOC);
        System.out.println("numDocs=" + numDocs);
        for (int i = 0; i < numDocs; i++) {
            final long t0 = System.currentTimeMillis();
            w.addDocument(doc);
            System.out.println(i + " of " + numDocs + " " + (System.currentTimeMillis() - t0) + " msec");
        }
        savedTerms = ts.savedTerms;
        System.out.println("TEST: full merge");
        w.forceMerge(1);
        System.out.println("TEST: close writer");
        w.close();
    }
    System.out.println("TEST: open reader");
    final IndexReader r = DirectoryReader.open(dir);
    if (savedTerms == null) {
        savedTerms = findTerms(r);
    }
    final int numSavedTerms = savedTerms.size();
    final List<BytesRef> bigOrdTerms = new ArrayList<>(savedTerms.subList(numSavedTerms - 10, numSavedTerms));
    System.out.println("TEST: test big ord terms...");
    testSavedTerms(r, bigOrdTerms);
    System.out.println("TEST: test all saved terms...");
    testSavedTerms(r, savedTerms);
    r.close();
    System.out.println("TEST: now CheckIndex...");
    CheckIndex.Status status = TestUtil.checkIndex(dir);
    final long tc = status.segmentInfos.get(0).termIndexStatus.termCount;
    assertTrue("count " + tc + " is not > " + Integer.MAX_VALUE, tc > Integer.MAX_VALUE);
    dir.close();
    System.out.println("TEST: done!");
}

Example 9 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class TestAddIndexes method testAddIndexMissingCodec.

/*
   * simple test that ensures we getting expected exceptions 
   */
public void testAddIndexMissingCodec() throws IOException {
    BaseDirectoryWrapper toAdd = newDirectory();
    // Disable checkIndex, else we get an exception because
    // of the unregistered codec:
    toAdd.setCheckIndexOnClose(false);
    {
        IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
        conf.setCodec(new UnRegisteredCodec());
        IndexWriter w = new IndexWriter(toAdd, conf);
        Document doc = new Document();
        FieldType customType = new FieldType();
        customType.setIndexOptions(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS);
        doc.add(newField("foo", "bar", customType));
        w.addDocument(doc);
        w.close();
    }
    {
        Directory dir = newDirectory();
        IndexWriterConfig conf = newIndexWriterConfig(new MockAnalyzer(random()));
        conf.setCodec(TestUtil.alwaysPostingsFormat(new MemoryPostingsFormat()));
        IndexWriter w = new IndexWriter(dir, conf);
        expectThrows(IllegalArgumentException.class, () -> {
            w.addIndexes(toAdd);
        });
        w.close();
        IndexReader open = DirectoryReader.open(dir);
        assertEquals(0, open.numDocs());
        open.close();
        dir.close();
    }
    expectThrows(IllegalArgumentException.class, () -> {
        DirectoryReader.open(toAdd);
    });
    toAdd.close();
}

Example 10 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class TestAllFilesDetectTruncation method truncateOneFile.

private void truncateOneFile(Directory dir, String victim) throws IOException {
    try (BaseDirectoryWrapper dirCopy = newDirectory()) {
        dirCopy.setCheckIndexOnClose(false);
        long victimLength = dir.fileLength(victim);
        int lostBytes = TestUtil.nextInt(random(), 1, (int) Math.min(100, victimLength));
        assert victimLength > 0;
        if (VERBOSE) {
            System.out.println("TEST: now truncate file " + victim + " by removing " + lostBytes + " of " + victimLength + " bytes");
        }
        for (String name : dir.listAll()) {
            if (name.equals(victim) == false) {
                dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
            } else {
                try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
                    IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
                    out.copyBytes(in, victimLength - lostBytes);
                }
            }
            dirCopy.sync(Collections.singleton(name));
        }
        try {
            // NOTE: we .close so that if the test fails (truncation not detected) we don't also get all these confusing errors about open files:
            DirectoryReader.open(dirCopy).close();
            fail("truncation not detected after removing " + lostBytes + " bytes out of " + victimLength + " for file " + victim);
        } catch (CorruptIndexException | EOFException e) {
        // expected
        }
        // CheckIndex should also fail:
        try {
            TestUtil.checkIndex(dirCopy, true, true, null);
            fail("truncation not detected after removing " + lostBytes + " bytes out of " + victimLength + " for file " + victim);
        } catch (CorruptIndexException | EOFException e) {
        // expected
        }
    }
}