Examples with FilterDirectory org.apache.lucene.store.FilterDirectory used on opensource projects

Example 1 with FilterDirectory

use of org.apache.lucene.store.FilterDirectory in project lucene-solr by apache.

the class TestIndexWriterMaxDocs method testAddTooManyIndexesDir.

/** 
   * LUCENE-6299: Test if addindexes(Dir[]) prevents exceeding max docs.
   */
public void testAddTooManyIndexesDir() throws Exception {
    // we cheat and add the same one over again... IW wants a write lock on each
    Directory dir = newDirectory(random(), NoLockFactory.INSTANCE);
    Document doc = new Document();
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
    for (int i = 0; i < 100000; i++) {
        w.addDocument(doc);
    }
    w.forceMerge(1);
    w.commit();
    w.close();
    // wrap this with disk full, so test fails faster and doesn't fill up real disks.
    MockDirectoryWrapper dir2 = newMockDirectory();
    w = new IndexWriter(dir2, new IndexWriterConfig(null));
    // don't confuse checkindex
    w.commit();
    // 64KB
    dir2.setMaxSizeInBytes(dir2.sizeInBytes() + 65536);
    Directory[] dirs = new Directory[1 + (IndexWriter.MAX_DOCS / 100000)];
    for (int i = 0; i < dirs.length; i++) {
        // bypass iw check for duplicate dirs
        dirs[i] = new FilterDirectory(dir) {
        };
    }
    try {
        w.addIndexes(dirs);
        fail("didn't get expected exception");
    } catch (IllegalArgumentException expected) {
    // pass
    } catch (IOException fakeDiskFull) {
        final Exception e;
        if (fakeDiskFull.getMessage() != null && fakeDiskFull.getMessage().startsWith("fake disk full")) {
            e = new RuntimeException("test failed: IW checks aren't working and we are executing addIndexes");
            e.addSuppressed(fakeDiskFull);
        } else {
            e = fakeDiskFull;
        }
        throw e;
    }
    w.close();
    dir.close();
    dir2.close();
}

Example 2 with FilterDirectory

use of org.apache.lucene.store.FilterDirectory in project lucene-solr by apache.

the class TestOfflineSorter method testBitFlippedOnPartition1.

/** Make sure corruption on a temp file (partition) is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnPartition1() throws Exception {
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            boolean corrupted;

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (corrupted == false && suffix.equals("sort")) {
                    corrupted = true;
                    return new CorruptingIndexOutput(dir0, 544677, out);
                } else {
                    return out;
                }
            }
        };
        IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
        writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));
        CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
            new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
        });
        assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
    }
}

Example 3 with FilterDirectory

use of org.apache.lucene.store.FilterDirectory in project lucene-solr by apache.

the class TestOfflineSorter method testBitFlippedOnPartition2.

/** Make sure corruption on a temp file (partition) is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnPartition2() throws Exception {
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            boolean corrupted;

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (corrupted == false && suffix.equals("sort")) {
                    corrupted = true;
                    return new CorruptingIndexOutput(dir0, 544677, out) {

                        @Override
                        protected void corruptFile() throws IOException {
                            String newTempName;
                            try (IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                                IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                                newTempName = tmpOut.getName();
                                tmpOut.copyBytes(in, 1025905);
                                short v = in.readShort();
                                assertEquals(254, v);
                                tmpOut.writeShort(Short.MAX_VALUE);
                                tmpOut.copyBytes(in, in.length() - 1025905 - Short.BYTES);
                            }
                            // Delete original and copy corrupt version back:
                            dir0.deleteFile(out.getName());
                            dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
                            dir0.deleteFile(newTempName);
                        }
                    };
                } else {
                    return out;
                }
            }
        };
        IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
        writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));
        EOFException e = expectThrows(EOFException.class, () -> {
            new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
        });
        assertEquals(1, e.getSuppressed().length);
        assertTrue(e.getSuppressed()[0] instanceof CorruptIndexException);
        assertTrue(e.getSuppressed()[0].getMessage().contains("checksum failed (hardware problem?)"));
    }
}

Example 4 with FilterDirectory

use of org.apache.lucene.store.FilterDirectory in project lucene-solr by apache.

the class TestOfflineSorter method testBitFlippedOnInput1.

/** Make sure corruption on the incoming (unsorted) file is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnInput1() throws Exception {
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (prefix.equals("unsorted")) {
                    return new CorruptingIndexOutput(dir0, 22, out);
                } else {
                    return out;
                }
            }
        };
        IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
        writeAll(unsorted, generateFixed(10 * 1024));
        CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
            new OfflineSorter(dir, "foo").sort(unsorted.getName());
        });
        assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
    }
}

Example 5 with FilterDirectory

use of org.apache.lucene.store.FilterDirectory in project lucene-solr by apache.

the class TestOfflineSorter method testBitFlippedOnInput2.

/** Make sure corruption on the incoming (unsorted) file is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnInput2() throws Exception {
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (prefix.equals("unsorted")) {
                    return new CorruptingIndexOutput(dir0, 22, out) {

                        @Override
                        protected void corruptFile() throws IOException {
                            String newTempName;
                            try (IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                                IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                                newTempName = tmpOut.getName();
                                // Replace length at the end with a too-long value:
                                short v = in.readShort();
                                assertEquals(256, v);
                                tmpOut.writeShort(Short.MAX_VALUE);
                                tmpOut.copyBytes(in, in.length() - Short.BYTES);
                            }
                            // Delete original and copy corrupt version back:
                            dir0.deleteFile(out.getName());
                            dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
                            dir0.deleteFile(newTempName);
                        }
                    };
                } else {
                    return out;
                }
            }
        };
        IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
        writeAll(unsorted, generateFixed(5 * 1024));
        // This corruption made OfflineSorter fail with its own exception, but we verify it also went and added (as suppressed) that the
        // checksum was wrong:
        EOFException e = expectThrows(EOFException.class, () -> {
            new OfflineSorter(dir, "foo").sort(unsorted.getName());
        });
        assertEquals(1, e.getSuppressed().length);
        assertTrue(e.getSuppressed()[0] instanceof CorruptIndexException);
        assertTrue(e.getSuppressed()[0].getMessage().contains("checksum failed (hardware problem?)"));
    }
}