Examples with IndexInput org.apache.lucene.store.IndexInput used on opensource projects

Example 21 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class BaseCompoundFormatTestCase method testRandomAccess.

/** This test opens two files from a compound stream and verifies that
   *  their file positions are independent of each other.
   */
public void testRandomAccess() throws IOException {
    Directory dir = newDirectory();
    Directory cr = createLargeCFS(dir);
    // Open two files
    IndexInput e1 = dir.openInput("_123.f11", newIOContext(random()));
    IndexInput e2 = dir.openInput("_123.f3", newIOContext(random()));
    IndexInput a1 = cr.openInput("_123.f11", newIOContext(random()));
    IndexInput a2 = dir.openInput("_123.f3", newIOContext(random()));
    // Seek the first pair
    e1.seek(100);
    a1.seek(100);
    assertEquals(100, e1.getFilePointer());
    assertEquals(100, a1.getFilePointer());
    byte be1 = e1.readByte();
    byte ba1 = a1.readByte();
    assertEquals(be1, ba1);
    // Now seek the second pair
    e2.seek(1027);
    a2.seek(1027);
    assertEquals(1027, e2.getFilePointer());
    assertEquals(1027, a2.getFilePointer());
    byte be2 = e2.readByte();
    byte ba2 = a2.readByte();
    assertEquals(be2, ba2);
    // Now make sure the first one didn't move
    assertEquals(101, e1.getFilePointer());
    assertEquals(101, a1.getFilePointer());
    be1 = e1.readByte();
    ba1 = a1.readByte();
    assertEquals(be1, ba1);
    // Now more the first one again, past the buffer length
    e1.seek(1910);
    a1.seek(1910);
    assertEquals(1910, e1.getFilePointer());
    assertEquals(1910, a1.getFilePointer());
    be1 = e1.readByte();
    ba1 = a1.readByte();
    assertEquals(be1, ba1);
    // Now make sure the second set didn't move
    assertEquals(1028, e2.getFilePointer());
    assertEquals(1028, a2.getFilePointer());
    be2 = e2.readByte();
    ba2 = a2.readByte();
    assertEquals(be2, ba2);
    // Move the second set back, again cross the buffer size
    e2.seek(17);
    a2.seek(17);
    assertEquals(17, e2.getFilePointer());
    assertEquals(17, a2.getFilePointer());
    be2 = e2.readByte();
    ba2 = a2.readByte();
    assertEquals(be2, ba2);
    // Finally, make sure the first set didn't move
    // Now make sure the first one didn't move
    assertEquals(1911, e1.getFilePointer());
    assertEquals(1911, a1.getFilePointer());
    be1 = e1.readByte();
    ba1 = a1.readByte();
    assertEquals(be1, ba1);
    e1.close();
    e2.close();
    a1.close();
    a2.close();
    cr.close();
    dir.close();
}

Example 22 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class IndexFetcher method compareFile.

protected static CompareResult compareFile(Directory indexDir, String filename, Long backupIndexFileLen, Long backupIndexFileChecksum) {
    CompareResult compareResult = new CompareResult();
    try {
        try (final IndexInput indexInput = indexDir.openInput(filename, IOContext.READONCE)) {
            long indexFileLen = indexInput.length();
            long indexFileChecksum = 0;
            if (backupIndexFileChecksum != null) {
                try {
                    indexFileChecksum = CodecUtil.retrieveChecksum(indexInput);
                    compareResult.checkSummed = true;
                } catch (Exception e) {
                    LOG.warn("Could not retrieve checksum from file.", e);
                }
            }
            if (!compareResult.checkSummed) {
                if (indexFileLen == backupIndexFileLen) {
                    compareResult.equal = true;
                    return compareResult;
                } else {
                    LOG.info("File {} did not match. expected length is {} and actual length is {}", filename, backupIndexFileLen, indexFileLen);
                    compareResult.equal = false;
                    return compareResult;
                }
            }
            if (indexFileLen == backupIndexFileLen && indexFileChecksum == backupIndexFileChecksum) {
                compareResult.equal = true;
                return compareResult;
            } else {
                LOG.warn("File {} did not match. expected checksum is {} and actual is checksum {}. " + "expected length is {} and actual length is {}", filename, backupIndexFileChecksum, indexFileChecksum, backupIndexFileLen, indexFileLen);
                compareResult.equal = false;
                return compareResult;
            }
        }
    } catch (NoSuchFileException | FileNotFoundException e) {
        compareResult.equal = false;
        return compareResult;
    } catch (IOException e) {
        LOG.error("Could not read file " + filename + ". Downloading it again", e);
        compareResult.equal = false;
        return compareResult;
    }
}

Example 23 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class TestOfflineSorter method assertFilesIdentical.

/**
   * Make sure two files are byte-byte identical.
   */
private void assertFilesIdentical(Directory dir, String golden, String sorted) throws IOException {
    long numBytes = dir.fileLength(golden);
    assertEquals(numBytes, dir.fileLength(sorted));
    byte[] buf1 = new byte[64 * 1024];
    byte[] buf2 = new byte[64 * 1024];
    try (IndexInput in1 = dir.openInput(golden, IOContext.READONCE);
        IndexInput in2 = dir.openInput(sorted, IOContext.READONCE)) {
        long left = numBytes;
        while (left > 0) {
            int chunk = (int) Math.min(buf1.length, left);
            left -= chunk;
            in1.readBytes(buf1, 0, chunk);
            in2.readBytes(buf2, 0, chunk);
            for (int i = 0; i < chunk; i++) {
                assertEquals(buf1[i], buf2[i]);
            }
        }
    }
}

Example 24 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class TestOfflineSorter method testBitFlippedOnPartition2.

/** Make sure corruption on a temp file (partition) is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnPartition2() throws Exception {
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            boolean corrupted;

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (corrupted == false && suffix.equals("sort")) {
                    corrupted = true;
                    return new CorruptingIndexOutput(dir0, 544677, out) {

                        @Override
                        protected void corruptFile() throws IOException {
                            String newTempName;
                            try (IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                                IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                                newTempName = tmpOut.getName();
                                tmpOut.copyBytes(in, 1025905);
                                short v = in.readShort();
                                assertEquals(254, v);
                                tmpOut.writeShort(Short.MAX_VALUE);
                                tmpOut.copyBytes(in, in.length() - 1025905 - Short.BYTES);
                            }
                            // Delete original and copy corrupt version back:
                            dir0.deleteFile(out.getName());
                            dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
                            dir0.deleteFile(newTempName);
                        }
                    };
                } else {
                    return out;
                }
            }
        };
        IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
        writeAll(unsorted, generateFixed((int) (OfflineSorter.MB * 3)));
        EOFException e = expectThrows(EOFException.class, () -> {
            new OfflineSorter(dir, "foo", OfflineSorter.DEFAULT_COMPARATOR, BufferSize.megabytes(1), 10, -1, null, 0).sort(unsorted.getName());
        });
        assertEquals(1, e.getSuppressed().length);
        assertTrue(e.getSuppressed()[0] instanceof CorruptIndexException);
        assertTrue(e.getSuppressed()[0].getMessage().contains("checksum failed (hardware problem?)"));
    }
}

Example 25 with IndexInput

use of org.apache.lucene.store.IndexInput in project lucene-solr by apache.

the class TestOfflineSorter method testBitFlippedOnInput2.

/** Make sure corruption on the incoming (unsorted) file is caught, if the corruption did confuse OfflineSorter! */
public void testBitFlippedOnInput2() throws Exception {
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (prefix.equals("unsorted")) {
                    return new CorruptingIndexOutput(dir0, 22, out) {

                        @Override
                        protected void corruptFile() throws IOException {
                            String newTempName;
                            try (IndexOutput tmpOut = dir0.createTempOutput("tmp", "tmp", IOContext.DEFAULT);
                                IndexInput in = dir0.openInput(out.getName(), IOContext.DEFAULT)) {
                                newTempName = tmpOut.getName();
                                // Replace length at the end with a too-long value:
                                short v = in.readShort();
                                assertEquals(256, v);
                                tmpOut.writeShort(Short.MAX_VALUE);
                                tmpOut.copyBytes(in, in.length() - Short.BYTES);
                            }
                            // Delete original and copy corrupt version back:
                            dir0.deleteFile(out.getName());
                            dir0.copyFrom(dir0, newTempName, out.getName(), IOContext.DEFAULT);
                            dir0.deleteFile(newTempName);
                        }
                    };
                } else {
                    return out;
                }
            }
        };
        IndexOutput unsorted = dir.createTempOutput("unsorted", "tmp", IOContext.DEFAULT);
        writeAll(unsorted, generateFixed(5 * 1024));
        // This corruption made OfflineSorter fail with its own exception, but we verify it also went and added (as suppressed) that the
        // checksum was wrong:
        EOFException e = expectThrows(EOFException.class, () -> {
            new OfflineSorter(dir, "foo").sort(unsorted.getName());
        });
        assertEquals(1, e.getSuppressed().length);
        assertTrue(e.getSuppressed()[0] instanceof CorruptIndexException);
        assertTrue(e.getSuppressed()[0].getMessage().contains("checksum failed (hardware problem?)"));
    }
}