Examples with BaseDirectoryWrapper org.apache.lucene.store.BaseDirectoryWrapper used on opensource projects

Example 31 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class TestAllFilesCheckIndexHeader method checkOneFile.

private void checkOneFile(Directory dir, String victim) throws IOException {
    // use ramdir explicit, as we do evil things like try to generate broken files, deletes must work.
    try (BaseDirectoryWrapper dirCopy = new MockDirectoryWrapper(random(), new RAMDirectory())) {
        dirCopy.setCheckIndexOnClose(false);
        long victimLength = dir.fileLength(victim);
        int wrongBytes = TestUtil.nextInt(random(), 1, (int) Math.min(100, victimLength));
        assert victimLength > 0;
        if (VERBOSE) {
            System.out.println("TEST: now break file " + victim + " by randomizing first " + wrongBytes + " of " + victimLength);
        }
        for (String name : dir.listAll()) {
            if (name.equals(victim) == false) {
                dirCopy.copyFrom(dir, name, name, IOContext.DEFAULT);
            } else {
                // time this will only require one iteration!
                while (true) {
                    try (IndexOutput out = dirCopy.createOutput(name, IOContext.DEFAULT);
                        IndexInput in = dir.openInput(name, IOContext.DEFAULT)) {
                        // keeps same file length, but replaces the first wrongBytes with random bytes:
                        byte[] bytes = new byte[wrongBytes];
                        random().nextBytes(bytes);
                        out.writeBytes(bytes, 0, bytes.length);
                        byte[] bytes2 = new byte[wrongBytes];
                        in.readBytes(bytes2, 0, bytes2.length);
                        if (Arrays.equals(bytes, bytes2) == false) {
                            // We successfully randomly generated bytes that differ from the bytes in the file:
                            out.copyBytes(in, victimLength - wrongBytes);
                            break;
                        }
                    }
                    // we have to try again, delete the first attempt and retry the loop
                    dirCopy.deleteFile(name);
                }
            }
            dirCopy.sync(Collections.singleton(name));
        }
        try {
            // NOTE: we .close so that if the test fails (truncation not detected) we don't also get all these confusing errors about open files:
            DirectoryReader.open(dirCopy).close();
            fail("wrong bytes not detected after randomizing first " + wrongBytes + " bytes out of " + victimLength + " for file " + victim);
        } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
        // expected
        }
        // CheckIndex should also fail:
        try {
            TestUtil.checkIndex(dirCopy, true, true, null);
            fail("wrong bytes not detected after randomizing first " + wrongBytes + " bytes out of " + victimLength + " for file " + victim);
        } catch (CorruptIndexException | EOFException | IndexFormatTooOldException e) {
        // expected
        }
    }
}

Example 32 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class Test2BPostings method test.

@Nightly
public void test() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BPostings"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriterConfig iwc = new IndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setRAMBufferSizeMB(256.0).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy(false, 10)).setOpenMode(IndexWriterConfig.OpenMode.CREATE);
    IndexWriter w = new IndexWriter(dir, iwc);
    MergePolicy mp = w.getConfig().getMergePolicy();
    if (mp instanceof LogByteSizeMergePolicy) {
        // 1 petabyte:
        ((LogByteSizeMergePolicy) mp).setMaxMergeMB(1024 * 1024 * 1024);
    }
    Document doc = new Document();
    FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
    ft.setOmitNorms(true);
    ft.setIndexOptions(IndexOptions.DOCS);
    Field field = new Field("field", new MyTokenStream(), ft);
    doc.add(field);
    final int numDocs = (Integer.MAX_VALUE / 26) + 1;
    for (int i = 0; i < numDocs; i++) {
        w.addDocument(doc);
        if (VERBOSE && i % 100000 == 0) {
            System.out.println(i + " of " + numDocs + "...");
        }
    }
    w.forceMerge(1);
    w.close();
    dir.close();
}

Example 33 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class Test2BSortedDocValuesFixedSorted method testFixedSorted.

// indexes Integer.MAX_VALUE docs with a fixed binary field
public void testFixedSorted() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BFixedSorted"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setRAMBufferSizeMB(256.0).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy(false, 10)).setOpenMode(IndexWriterConfig.OpenMode.CREATE).setCodec(TestUtil.getDefaultCodec()));
    Document doc = new Document();
    byte[] bytes = new byte[2];
    BytesRef data = new BytesRef(bytes);
    SortedDocValuesField dvField = new SortedDocValuesField("dv", data);
    doc.add(dvField);
    for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
        bytes[0] = (byte) (i >> 8);
        bytes[1] = (byte) i;
        w.addDocument(doc);
        if (i % 100000 == 0) {
            System.out.println("indexed: " + i);
            System.out.flush();
        }
    }
    w.forceMerge(1);
    w.close();
    System.out.println("verifying...");
    System.out.flush();
    DirectoryReader r = DirectoryReader.open(dir);
    int expectedValue = 0;
    for (LeafReaderContext context : r.leaves()) {
        LeafReader reader = context.reader();
        BinaryDocValues dv = DocValues.getBinary(reader, "dv");
        for (int i = 0; i < reader.maxDoc(); i++) {
            assertEquals(i, dv.nextDoc());
            bytes[0] = (byte) (expectedValue >> 8);
            bytes[1] = (byte) expectedValue;
            final BytesRef term = dv.binaryValue();
            assertEquals(data, term);
            expectedValue++;
        }
    }
    r.close();
    dir.close();
}

Example 34 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class Test2BSortedDocValuesOrds method test2BOrds.

// indexes Integer.MAX_VALUE docs with a fixed binary field
public void test2BOrds() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BOrds"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setRAMBufferSizeMB(256.0).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy(false, 10)).setOpenMode(IndexWriterConfig.OpenMode.CREATE).setCodec(TestUtil.getDefaultCodec()));
    Document doc = new Document();
    byte[] bytes = new byte[4];
    BytesRef data = new BytesRef(bytes);
    SortedDocValuesField dvField = new SortedDocValuesField("dv", data);
    doc.add(dvField);
    for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
        bytes[0] = (byte) (i >> 24);
        bytes[1] = (byte) (i >> 16);
        bytes[2] = (byte) (i >> 8);
        bytes[3] = (byte) i;
        w.addDocument(doc);
        if (i % 100000 == 0) {
            System.out.println("indexed: " + i);
            System.out.flush();
        }
    }
    w.forceMerge(1);
    w.close();
    System.out.println("verifying...");
    System.out.flush();
    DirectoryReader r = DirectoryReader.open(dir);
    int counter = 0;
    for (LeafReaderContext context : r.leaves()) {
        LeafReader reader = context.reader();
        BytesRef scratch = new BytesRef();
        BinaryDocValues dv = DocValues.getBinary(reader, "dv");
        for (int i = 0; i < reader.maxDoc(); i++) {
            assertEquals(i, dv.nextDoc());
            bytes[0] = (byte) (counter >> 24);
            bytes[1] = (byte) (counter >> 16);
            bytes[2] = (byte) (counter >> 8);
            bytes[3] = (byte) counter;
            counter++;
            final BytesRef term = dv.binaryValue();
            assertEquals(data, term);
        }
    }
    r.close();
    dir.close();
}

Example 35 with BaseDirectoryWrapper

use of org.apache.lucene.store.BaseDirectoryWrapper in project lucene-solr by apache.

the class Test2BBinaryDocValues method testVariableBinary.

// indexes IndexWriter.MAX_DOCS docs with a variable binary field
public void testVariableBinary() throws Exception {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("2BVariableBinary"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(new MockAnalyzer(random())).setMaxBufferedDocs(IndexWriterConfig.DISABLE_AUTO_FLUSH).setRAMBufferSizeMB(256.0).setMergeScheduler(new ConcurrentMergeScheduler()).setMergePolicy(newLogMergePolicy(false, 10)).setOpenMode(IndexWriterConfig.OpenMode.CREATE).setCodec(TestUtil.getDefaultCodec()));
    Document doc = new Document();
    byte[] bytes = new byte[4];
    ByteArrayDataOutput encoder = new ByteArrayDataOutput(bytes);
    BytesRef data = new BytesRef(bytes);
    BinaryDocValuesField dvField = new BinaryDocValuesField("dv", data);
    doc.add(dvField);
    for (int i = 0; i < IndexWriter.MAX_DOCS; i++) {
        encoder.reset(bytes);
        // 1, 2, or 3 bytes
        encoder.writeVInt(i % 65535);
        data.length = encoder.getPosition();
        w.addDocument(doc);
        if (i % 100000 == 0) {
            System.out.println("indexed: " + i);
            System.out.flush();
        }
    }
    w.forceMerge(1);
    w.close();
    System.out.println("verifying...");
    System.out.flush();
    DirectoryReader r = DirectoryReader.open(dir);
    int expectedValue = 0;
    ByteArrayDataInput input = new ByteArrayDataInput();
    for (LeafReaderContext context : r.leaves()) {
        LeafReader reader = context.reader();
        BinaryDocValues dv = reader.getBinaryDocValues("dv");
        for (int i = 0; i < reader.maxDoc(); i++) {
            assertEquals(i, dv.nextDoc());
            final BytesRef term = dv.binaryValue();
            input.reset(term.bytes, term.offset, term.length);
            assertEquals(expectedValue % 65535, input.readVInt());
            assertTrue(input.eof());
            expectedValue++;
        }
    }
    r.close();
    dir.close();
}