Examples with IndexOutput org.apache.lucene.store.IndexOutput used on opensource projects

Example 21 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestOfflineSorter method testBitFlippedOnPartition1.

/** Make sure corruption on a temp file (partition) is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnPartition1() throws Exception {
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            boolean corrupted;

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (corrupted == false && suffix.equals("sort")) {
                    corrupted = true;
                    return new CorruptingIndexOutput(dir0, 544677, out);
                } else {
                    return out;
                }
            }
        };
        IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
        writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));
        CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
            new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
        });
        assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
    }
}

Example 22 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestOfflineSorter method testBitFlippedOnPartition2.

/** Make sure corruption on a temp file (partition) is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnPartition2() throws Exception {
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            boolean corrupted;

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (corrupted == false && suffix.equals("sort")) {
                    corrupted = true;
                    return new CorruptingIndexOutput(dir0, 544677, out) {

                        @Override
                        protected void corruptFile() throws IOException {
                            String newTempName;
                            try (IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                                IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                                newTempName = tmpOut.getName();
                                tmpOut.copyBytes(in, 1025905);
                                short v = in.readShort();
                                assertEquals(254, v);
                                tmpOut.writeShort(Short.MAX_VALUE);
                                tmpOut.copyBytes(in, in.length() - 1025905 - Short.BYTES);
                            }
                            // Delete original and copy corrupt version back:
                            dir0.deleteFile(out.getName());
                            dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
                            dir0.deleteFile(newTempName);
                        }
                    };
                } else {
                    return out;
                }
            }
        };
        IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
        writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));
        EOFException e = expectThrows(EOFException.class, () -> {
            new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
        });
        assertEquals(1, e.getSuppressed().length);
        assertTrue(e.getSuppressed()[0] instanceof CorruptIndexException);
        assertTrue(e.getSuppressed()[0].getMessage().contains("checksum failed (hardware problem?)"));
    }
}

Example 23 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestOfflineSorter method testBitFlippedOnInput1.

/** Make sure corruption on the incoming (unsorted) file is caught, even if the corruption didn't confuse OfflineSorter! */
public void testBitFlippedOnInput1() throws Exception {
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (prefix.equals("unsorted")) {
                    return new CorruptingIndexOutput(dir0, 22, out);
                } else {
                    return out;
                }
            }
        };
        IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
        writeAll(unsorted, generateFixed(10 * 1024));
        CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
            new OfflineSorter(dir, "foo").sort(unsorted.getName());
        });
        assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
    }
}

Example 24 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestOfflineSorter method testBitFlippedOnInput2.

/** Make sure corruption on the incoming (unsorted) file is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnInput2() throws Exception {
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (prefix.equals("unsorted")) {
                    return new CorruptingIndexOutput(dir0, 22, out) {

                        @Override
                        protected void corruptFile() throws IOException {
                            String newTempName;
                            try (IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                                IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                                newTempName = tmpOut.getName();
                                // Replace length at the end with a too-long value:
                                short v = in.readShort();
                                assertEquals(256, v);
                                tmpOut.writeShort(Short.MAX_VALUE);
                                tmpOut.copyBytes(in, in.length() - Short.BYTES);
                            }
                            // Delete original and copy corrupt version back:
                            dir0.deleteFile(out.getName());
                            dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
                            dir0.deleteFile(newTempName);
                        }
                    };
                } else {
                    return out;
                }
            }
        };
        IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
        writeAll(unsorted, generateFixed(5 * 1024));
        // This corruption made OfflineSorter fail with its own exception, but we verify it also went and added (as suppressed) that the
        // checksum was wrong:
        EOFException e = expectThrows(EOFException.class, () -> {
            new OfflineSorter(dir, "foo").sort(unsorted.getName());
        });
        assertEquals(1, e.getSuppressed().length);
        assertTrue(e.getSuppressed()[0] instanceof CorruptIndexException);
        assertTrue(e.getSuppressed()[0].getMessage().contains("checksum failed (hardware problem?)"));
    }
}

Example 25 with IndexOutput

use of org.apache.lucene.store.IndexOutput in project lucene-solr by apache.

the class TestPagedBytes method testOverflow.

// memory hole
@Ignore
public void testOverflow() throws IOException {
    BaseDirectoryWrapper dir = newFSDirectory(createTempDir("testOverflow"));
    if (dir instanceof MockDirectoryWrapper) {
        ((MockDirectoryWrapper) dir).setThrottling(MockDirectoryWrapper.Throttling.NEVER);
    }
    final int blockBits = TestUtil.nextInt(random(), 14, 28);
    final int blockSize = 1 << blockBits;
    byte[] arr = new byte[TestUtil.nextInt(random(), blockSize / 2, blockSize * 2)];
    for (int i = 0; i < arr.length; ++i) {
        arr[i] = (byte) i;
    }
    final long numBytes = (1L << 31) + TestUtil.nextInt(random(), 1, blockSize * 3);
    final PagedBytes p = new PagedBytes(blockBits);
    final IndexOutput out = dir.createOutput("foo", IOContext.DEFAULT);
    for (long i = 0; i < numBytes; ) {
        assertEquals(i, out.getFilePointer());
        final int len = (int) Math.min(arr.length, numBytes - i);
        out.writeBytes(arr, len);
        i += len;
    }
    assertEquals(numBytes, out.getFilePointer());
    out.close();
    final IndexInput in = dir.openInput("foo", IOContext.DEFAULT);
    p.copy(in, numBytes);
    final PagedBytes.Reader reader = p.freeze(random().nextBoolean());
    for (long offset : new long[] { 0L, Integer.MAX_VALUE, numBytes - 1, TestUtil.nextLong(random(), 1, numBytes - 2) }) {
        BytesRef b = new BytesRef();
        reader.fillSlice(b, offset, 1);
        assertEquals(arr[(int) (offset % arr.length)], b.bytes[b.offset]);
    }
    in.close();
    dir.close();
}