Examples with ChecksumIndexInput org.apache.lucene.store.ChecksumIndexInput used on opensource projects

Example 1 with ChecksumIndexInput

use of org.apache.lucene.store.ChecksumIndexInput in project elasticsearch by elastic.

the class Store method failIfCorrupted.

private static void failIfCorrupted(Directory directory, ShardId shardId) throws IOException {
    final String[] files = directory.listAll();
    List<CorruptIndexException> ex = new ArrayList<>();
    for (String file : files) {
        if (file.startsWith(CORRUPTED)) {
            try (ChecksumIndexInput input = directory.openChecksumInput(file, IOContext.READONCE)) {
                int version = CodecUtil.checkHeader(input, CODEC, VERSION_START, VERSION);
                if (version == VERSION_WRITE_THROWABLE) {
                    final int size = input.readVInt();
                    final byte[] buffer = new byte[size];
                    input.readBytes(buffer, 0, buffer.length);
                    StreamInput in = StreamInput.wrap(buffer);
                    Exception t = in.readException();
                    if (t instanceof CorruptIndexException) {
                        ex.add((CorruptIndexException) t);
                    } else {
                        ex.add(new CorruptIndexException(t.getMessage(), "preexisting_corruption", t));
                    }
                } else {
                    assert version == VERSION_START || version == VERSION_STACK_TRACE;
                    String msg = input.readString();
                    StringBuilder builder = new StringBuilder(shardId.toString());
                    builder.append(" Preexisting corrupted index [");
                    builder.append(file).append("] caused by: ");
                    builder.append(msg);
                    if (version == VERSION_STACK_TRACE) {
                        builder.append(System.lineSeparator());
                        builder.append(input.readString());
                    }
                    ex.add(new CorruptIndexException(builder.toString(), "preexisting_corruption"));
                }
                CodecUtil.checkFooter(input);
            }
        }
    }
    if (ex.isEmpty() == false) {
        ExceptionsHelper.rethrowAndSuppress(ex);
    }
}

Example 2 with ChecksumIndexInput

use of org.apache.lucene.store.ChecksumIndexInput in project elasticsearch by elastic.

the class MetaDataStateFormatTests method corruptFile.

public static void corruptFile(Path file, Logger logger) throws IOException {
    Path fileToCorrupt = file;
    try (SimpleFSDirectory dir = new SimpleFSDirectory(fileToCorrupt.getParent())) {
        long checksumBeforeCorruption;
        try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
            checksumBeforeCorruption = CodecUtil.retrieveChecksum(input);
        }
        try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
            raf.position(randomIntBetween(0, (int) Math.min(Integer.MAX_VALUE, raf.size() - 1)));
            long filePointer = raf.position();
            ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
            raf.read(bb);
            bb.flip();
            byte oldValue = bb.get(0);
            byte newValue = (byte) ~oldValue;
            bb.put(0, newValue);
            raf.write(bb, filePointer);
            logger.debug("Corrupting file {} --  flipping at position {} from {} to {} ", fileToCorrupt.getFileName().toString(), filePointer, Integer.toHexString(oldValue), Integer.toHexString(newValue));
        }
        long checksumAfterCorruption;
        long actualChecksumAfterCorruption;
        try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
            assertThat(input.getFilePointer(), is(0L));
            // one long is the checksum... 8 bytes
            input.seek(input.length() - 8);
            checksumAfterCorruption = input.getChecksum();
            actualChecksumAfterCorruption = input.readLong();
        }
        StringBuilder msg = new StringBuilder();
        msg.append("Checksum before: [").append(checksumBeforeCorruption).append("]");
        msg.append(" after: [").append(checksumAfterCorruption).append("]");
        msg.append(" checksum value after corruption: ").append(actualChecksumAfterCorruption).append("]");
        msg.append(" file: ").append(fileToCorrupt.getFileName().toString()).append(" length: ").append(dir.fileLength(fileToCorrupt.getFileName().toString()));
        logger.debug("{}", msg.toString());
        assumeTrue("Checksum collision - " + msg.toString(), // collision
        checksumAfterCorruption != checksumBeforeCorruption || // checksum corrupted
        actualChecksumAfterCorruption != checksumBeforeCorruption);
    }
}

Example 3 with ChecksumIndexInput

use of org.apache.lucene.store.ChecksumIndexInput in project elasticsearch by elastic.

the class StoreTests method testVerifyingIndexInput.

public void testVerifyingIndexInput() throws IOException {
    Directory dir = newDirectory();
    IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
    int iters = scaledRandomIntBetween(10, 100);
    for (int i = 0; i < iters; i++) {
        BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
        output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
    }
    CodecUtil.writeFooter(output);
    output.close();
    // Check file
    IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
    long checksum = CodecUtil.retrieveChecksum(indexInput);
    indexInput.seek(0);
    IndexInput verifyingIndexInput = new Store.VerifyingIndexInput(dir.openInput("foo.bar", IOContext.DEFAULT));
    readIndexInputFullyWithRandomSeeks(verifyingIndexInput);
    Store.verify(verifyingIndexInput);
    assertThat(checksum, equalTo(((ChecksumIndexInput) verifyingIndexInput).getChecksum()));
    IOUtils.close(indexInput, verifyingIndexInput);
    // Corrupt file and check again
    corruptFile(dir, "foo.bar", "foo1.bar");
    verifyingIndexInput = new Store.VerifyingIndexInput(dir.openInput("foo1.bar", IOContext.DEFAULT));
    readIndexInputFullyWithRandomSeeks(verifyingIndexInput);
    try {
        Store.verify(verifyingIndexInput);
        fail("should be a corrupted index");
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
    // ok
    }
    IOUtils.close(verifyingIndexInput);
    IOUtils.close(dir);
}

Example 4 with ChecksumIndexInput

use of org.apache.lucene.store.ChecksumIndexInput in project elasticsearch by elastic.

the class KeyStoreWrapper method load.

/**
     * Loads information about the Elasticsearch keystore from the provided config directory.
     *
     * {@link #decrypt(char[])} must be called before reading or writing any entries.
     * Returns {@code null} if no keystore exists.
     */
public static KeyStoreWrapper load(Path configDir) throws IOException {
    Path keystoreFile = keystorePath(configDir);
    if (Files.exists(keystoreFile) == false) {
        return null;
    }
    SimpleFSDirectory directory = new SimpleFSDirectory(configDir);
    try (IndexInput indexInput = directory.openInput(KEYSTORE_FILENAME, IOContext.READONCE)) {
        ChecksumIndexInput input = new BufferedChecksumIndexInput(indexInput);
        CodecUtil.checkHeader(input, KEYSTORE_FILENAME, FORMAT_VERSION, FORMAT_VERSION);
        byte hasPasswordByte = input.readByte();
        boolean hasPassword = hasPasswordByte == 1;
        if (hasPassword == false && hasPasswordByte != 0) {
            throw new IllegalStateException("hasPassword boolean is corrupt: " + String.format(Locale.ROOT, "%02x", hasPasswordByte));
        }
        String type = input.readString();
        String secretKeyAlgo = input.readString();
        byte[] keystoreBytes = new byte[input.readInt()];
        input.readBytes(keystoreBytes, 0, keystoreBytes.length);
        CodecUtil.checkFooter(input);
        return new KeyStoreWrapper(hasPassword, type, secretKeyAlgo, keystoreBytes);
    }
}

Example 5 with ChecksumIndexInput

use of org.apache.lucene.store.ChecksumIndexInput in project elasticsearch by elastic.

the class CorruptionUtils method corruptFile.

/**
     * Corrupts a random file at a random position
     */
public static void corruptFile(Random random, Path... files) throws IOException {
    assertTrue("files must be non-empty", files.length > 0);
    final Path fileToCorrupt = RandomPicks.randomFrom(random, files);
    assertTrue(fileToCorrupt + " is not a file", Files.isRegularFile(fileToCorrupt));
    try (Directory dir = FSDirectory.open(fileToCorrupt.toAbsolutePath().getParent())) {
        long checksumBeforeCorruption;
        try (IndexInput input = dir.openInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
            checksumBeforeCorruption = CodecUtil.retrieveChecksum(input);
        }
        try (FileChannel raf = FileChannel.open(fileToCorrupt, StandardOpenOption.READ, StandardOpenOption.WRITE)) {
            // read
            raf.position(random.nextInt((int) Math.min(Integer.MAX_VALUE, raf.size())));
            long filePointer = raf.position();
            ByteBuffer bb = ByteBuffer.wrap(new byte[1]);
            raf.read(bb);
            bb.flip();
            // corrupt
            byte oldValue = bb.get(0);
            byte newValue = (byte) (oldValue + 1);
            bb.put(0, newValue);
            // rewrite
            raf.position(filePointer);
            raf.write(bb);
            logger.info("Corrupting file --  flipping at position {} from {} to {} file: {}", filePointer, Integer.toHexString(oldValue), Integer.toHexString(newValue), fileToCorrupt.getFileName());
        }
        long checksumAfterCorruption;
        long actualChecksumAfterCorruption;
        try (ChecksumIndexInput input = dir.openChecksumInput(fileToCorrupt.getFileName().toString(), IOContext.DEFAULT)) {
            assertThat(input.getFilePointer(), is(0L));
            // one long is the checksum... 8 bytes
            input.seek(input.length() - 8);
            checksumAfterCorruption = input.getChecksum();
            actualChecksumAfterCorruption = input.readLong();
        }
        // we need to add assumptions here that the checksums actually really don't match there is a small chance to get collisions
        // in the checksum which is ok though....
        StringBuilder msg = new StringBuilder();
        msg.append("before: [").append(checksumBeforeCorruption).append("] ");
        msg.append("after: [").append(checksumAfterCorruption).append("] ");
        msg.append("checksum value after corruption: ").append(actualChecksumAfterCorruption).append("] ");
        msg.append("file: ").append(fileToCorrupt.getFileName()).append(" length: ").append(dir.fileLength(fileToCorrupt.getFileName().toString()));
        logger.info("Checksum {}", msg);
        assumeTrue("Checksum collision - " + msg.toString(), // collision
        checksumAfterCorruption != checksumBeforeCorruption || // checksum corrupted
        actualChecksumAfterCorruption != checksumBeforeCorruption);
        assertThat("no file corrupted", fileToCorrupt, notNullValue());
    }
}