Examples with IndexFormatTooOldException org.apache.lucene.index.IndexFormatTooOldException used on opensource projects

Example 21 with IndexFormatTooOldException

use of org.apache.lucene.index.IndexFormatTooOldException in project crate by crate.

the class StoreTests method testVerifyingIndexOutput.

public void testVerifyingIndexOutput() throws IOException {
    Directory dir = newDirectory();
    IndexOutput output = dir.createOutput("foo.bar", IOContext.DEFAULT);
    int iters = scaledRandomIntBetween(10, 100);
    for (int i = 0; i < iters; i++) {
        BytesRef bytesRef = new BytesRef(TestUtil.randomRealisticUnicodeString(random(), 10, 1024));
        output.writeBytes(bytesRef.bytes, bytesRef.offset, bytesRef.length);
    }
    CodecUtil.writeFooter(output);
    output.close();
    IndexInput indexInput = dir.openInput("foo.bar", IOContext.DEFAULT);
    String checksum = Store.digestToString(CodecUtil.retrieveChecksum(indexInput));
    indexInput.seek(0);
    BytesRef ref = new BytesRef(scaledRandomIntBetween(1, 1024));
    long length = indexInput.length();
    IndexOutput verifyingOutput = new Store.LuceneVerifyingIndexOutput(new StoreFileMetadata("foo1.bar", length, checksum, MIN_SUPPORTED_LUCENE_VERSION), dir.createOutput("foo1.bar", IOContext.DEFAULT));
    while (length > 0) {
        if (random().nextInt(10) == 0) {
            verifyingOutput.writeByte(indexInput.readByte());
            length--;
        } else {
            int min = (int) Math.min(length, ref.bytes.length);
            indexInput.readBytes(ref.bytes, ref.offset, min);
            verifyingOutput.writeBytes(ref.bytes, ref.offset, min);
            length -= min;
        }
    }
    Store.verify(verifyingOutput);
    try {
        appendRandomData(verifyingOutput);
        fail("should be a corrupted index");
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
    // ok
    }
    try {
        Store.verify(verifyingOutput);
        fail("should be a corrupted index");
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
    // ok
    }
    IOUtils.close(indexInput, verifyingOutput, dir);
}

Example 22 with IndexFormatTooOldException

use of org.apache.lucene.index.IndexFormatTooOldException in project crate by crate.

the class RecoveryTarget method cleanFiles.

@Override
public void cleanFiles(int totalTranslogOps, long globalCheckpoint, Store.MetadataSnapshot sourceMetaData, ActionListener<Void> listener) {
    ActionListener.completeWith(listener, () -> {
        state().getTranslog().totalOperations(totalTranslogOps);
        // first, we go and move files that were created with the recovery id suffix to
        // the actual names, its ok if we have a corrupted index here, since we have replicas
        // to recover from in case of a full cluster shutdown just when this code executes...
        multiFileWriter.renameAllTempFiles();
        final Store store = store();
        store.incRef();
        try {
            store.cleanupAndVerify("recovery CleanFilesRequestHandler", sourceMetaData);
            final String translogUUID = Translog.createEmptyTranslog(indexShard.shardPath().resolveTranslog(), globalCheckpoint, shardId, indexShard.getPendingPrimaryTerm());
            store.associateIndexWithNewTranslog(translogUUID);
            if (indexShard.getRetentionLeases().leases().isEmpty()) {
                // if empty, may be a fresh IndexShard, so write an empty leases file to disk
                indexShard.persistRetentionLeases();
                assert indexShard.loadRetentionLeases().leases().isEmpty();
            } else {
                assert indexShard.assertRetentionLeasesPersisted();
            }
            indexShard.maybeCheckIndex();
            state().setStage(RecoveryState.Stage.TRANSLOG);
        } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) {
            // its content on disk if possible.
            try {
                try {
                    store.removeCorruptionMarker();
                } finally {
                    // clean up and delete all files
                    Lucene.cleanLuceneIndex(store.directory());
                }
            } catch (Exception e) {
                logger.debug("Failed to clean lucene index", e);
                ex.addSuppressed(e);
            }
            RecoveryFailedException rfe = new RecoveryFailedException(state(), "failed to clean after recovery", ex);
            fail(rfe, true);
            throw rfe;
        } catch (Exception ex) {
            RecoveryFailedException rfe = new RecoveryFailedException(state(), "failed to clean after recovery", ex);
            fail(rfe, true);
            throw rfe;
        } finally {
            store.decRef();
        }
        return null;
    });
}

Example 23 with IndexFormatTooOldException

use of org.apache.lucene.index.IndexFormatTooOldException in project elasticsearch by elastic.

the class TranslogReader method open.

/**
     * Given a file channel, opens a {@link TranslogReader}, taking care of checking and validating the file header.
     *
     * @param channel the translog file channel
     * @param path the path to the translog
     * @param checkpoint the translog checkpoint
     * @param translogUUID the tranlog UUID
     * @return a new TranslogReader
     * @throws IOException if any of the file operations resulted in an I/O exception
     */
public static TranslogReader open(final FileChannel channel, final Path path, final Checkpoint checkpoint, final String translogUUID) throws IOException {
    try {
        // don't close
        InputStreamStreamInput headerStream = new InputStreamStreamInput(java.nio.channels.Channels.newInputStream(channel));
        // Lucene's CodecUtil writes a magic number of 0x3FD76C17 with the
        // header, in binary this looks like:
        //
        // binary: 0011 1111 1101 0111 0110 1100 0001 0111
        // hex   :    3    f    d    7    6    c    1    7
        //
        // With version 0 of the translog, the first byte is the
        // Operation.Type, which will always be between 0-4, so we know if
        // we grab the first byte, it can be:
        // 0x3f => Lucene's magic number, so we can assume it's version 1 or later
        // 0x00 => version 0 of the translog
        //
        // otherwise the first byte of the translog is corrupted and we
        // should bail
        byte b1 = headerStream.readByte();
        if (b1 == LUCENE_CODEC_HEADER_BYTE) {
            // Read 3 more bytes, meaning a whole integer has been read
            byte b2 = headerStream.readByte();
            byte b3 = headerStream.readByte();
            byte b4 = headerStream.readByte();
            // Convert the 4 bytes that were read into an integer
            int header = ((b1 & 0xFF) << 24) + ((b2 & 0xFF) << 16) + ((b3 & 0xFF) << 8) + ((b4 & 0xFF) << 0);
            // byte separately
            if (header != CodecUtil.CODEC_MAGIC) {
                throw new TranslogCorruptedException("translog looks like version 1 or later, but has corrupted header. path:" + path);
            }
            // Confirm the rest of the header using CodecUtil, extracting
            // the translog version
            int version = CodecUtil.checkHeaderNoMagic(new InputStreamDataInput(headerStream), TranslogWriter.TRANSLOG_CODEC, 1, Integer.MAX_VALUE);
            switch(version) {
                case TranslogWriter.VERSION_CHECKSUMS:
                    throw new IllegalStateException("pre-2.0 translog found [" + path + "]");
                case TranslogWriter.VERSION_CHECKPOINTS:
                    assert path.getFileName().toString().endsWith(Translog.TRANSLOG_FILE_SUFFIX) : "new file ends with old suffix: " + path;
                    assert checkpoint.numOps >= 0 : "expected at least 0 operatin but got: " + checkpoint.numOps;
                    assert checkpoint.offset <= channel.size() : "checkpoint is inconsistent with channel length: " + channel.size() + " " + checkpoint;
                    int len = headerStream.readInt();
                    if (len > channel.size()) {
                        throw new TranslogCorruptedException("uuid length can't be larger than the translog");
                    }
                    BytesRef ref = new BytesRef(len);
                    ref.length = len;
                    headerStream.read(ref.bytes, ref.offset, ref.length);
                    BytesRef uuidBytes = new BytesRef(translogUUID);
                    if (uuidBytes.bytesEquals(ref) == false) {
                        throw new TranslogCorruptedException("expected shard UUID " + uuidBytes + " but got: " + ref + " this translog file belongs to a different translog. path:" + path);
                    }
                    final long firstOperationOffset = ref.length + CodecUtil.headerLength(TranslogWriter.TRANSLOG_CODEC) + Integer.BYTES;
                    return new TranslogReader(checkpoint, channel, path, firstOperationOffset);
                default:
                    throw new TranslogCorruptedException("No known translog stream version: " + version + " path:" + path);
            }
        } else if (b1 == UNVERSIONED_TRANSLOG_HEADER_BYTE) {
            throw new IllegalStateException("pre-1.4 translog found [" + path + "]");
        } else {
            throw new TranslogCorruptedException("Invalid first byte in translog file, got: " + Long.toHexString(b1) + ", expected 0x00 or 0x3f. path:" + path);
        }
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
        throw new TranslogCorruptedException("Translog header corrupted. path:" + path, e);
    }
}

Example 24 with IndexFormatTooOldException

use of org.apache.lucene.index.IndexFormatTooOldException in project elasticsearch by elastic.

the class StreamOutput method writeException.

public void writeException(Throwable throwable) throws IOException {
    if (throwable == null) {
        writeBoolean(false);
    } else {
        writeBoolean(true);
        boolean writeCause = true;
        boolean writeMessage = true;
        if (throwable instanceof CorruptIndexException) {
            writeVInt(1);
            writeOptionalString(((CorruptIndexException) throwable).getOriginalMessage());
            writeOptionalString(((CorruptIndexException) throwable).getResourceDescription());
            writeMessage = false;
        } else if (throwable instanceof IndexFormatTooNewException) {
            writeVInt(2);
            writeOptionalString(((IndexFormatTooNewException) throwable).getResourceDescription());
            writeInt(((IndexFormatTooNewException) throwable).getVersion());
            writeInt(((IndexFormatTooNewException) throwable).getMinVersion());
            writeInt(((IndexFormatTooNewException) throwable).getMaxVersion());
            writeMessage = false;
            writeCause = false;
        } else if (throwable instanceof IndexFormatTooOldException) {
            writeVInt(3);
            IndexFormatTooOldException t = (IndexFormatTooOldException) throwable;
            writeOptionalString(t.getResourceDescription());
            if (t.getVersion() == null) {
                writeBoolean(false);
                writeOptionalString(t.getReason());
            } else {
                writeBoolean(true);
                writeInt(t.getVersion());
                writeInt(t.getMinVersion());
                writeInt(t.getMaxVersion());
            }
            writeMessage = false;
            writeCause = false;
        } else if (throwable instanceof NullPointerException) {
            writeVInt(4);
            writeCause = false;
        } else if (throwable instanceof NumberFormatException) {
            writeVInt(5);
            writeCause = false;
        } else if (throwable instanceof IllegalArgumentException) {
            writeVInt(6);
        } else if (throwable instanceof AlreadyClosedException) {
            writeVInt(7);
        } else if (throwable instanceof EOFException) {
            writeVInt(8);
            writeCause = false;
        } else if (throwable instanceof SecurityException) {
            writeVInt(9);
        } else if (throwable instanceof StringIndexOutOfBoundsException) {
            writeVInt(10);
            writeCause = false;
        } else if (throwable instanceof ArrayIndexOutOfBoundsException) {
            writeVInt(11);
            writeCause = false;
        } else if (throwable instanceof FileNotFoundException) {
            writeVInt(12);
            writeCause = false;
        } else if (throwable instanceof FileSystemException) {
            writeVInt(13);
            if (throwable instanceof NoSuchFileException) {
                writeVInt(0);
            } else if (throwable instanceof NotDirectoryException) {
                writeVInt(1);
            } else if (throwable instanceof DirectoryNotEmptyException) {
                writeVInt(2);
            } else if (throwable instanceof AtomicMoveNotSupportedException) {
                writeVInt(3);
            } else if (throwable instanceof FileAlreadyExistsException) {
                writeVInt(4);
            } else if (throwable instanceof AccessDeniedException) {
                writeVInt(5);
            } else if (throwable instanceof FileSystemLoopException) {
                writeVInt(6);
            } else {
                writeVInt(7);
            }
            writeOptionalString(((FileSystemException) throwable).getFile());
            writeOptionalString(((FileSystemException) throwable).getOtherFile());
            writeOptionalString(((FileSystemException) throwable).getReason());
            writeCause = false;
        } else if (throwable instanceof IllegalStateException) {
            writeVInt(14);
        } else if (throwable instanceof LockObtainFailedException) {
            writeVInt(15);
        } else if (throwable instanceof InterruptedException) {
            writeVInt(16);
            writeCause = false;
        } else if (throwable instanceof IOException) {
            writeVInt(17);
        } else {
            ElasticsearchException ex;
            if (throwable instanceof ElasticsearchException && ElasticsearchException.isRegistered(throwable.getClass(), version)) {
                ex = (ElasticsearchException) throwable;
            } else {
                ex = new NotSerializableExceptionWrapper(throwable);
            }
            writeVInt(0);
            writeVInt(ElasticsearchException.getId(ex.getClass()));
            ex.writeTo(this);
            return;
        }
        if (writeMessage) {
            writeOptionalString(throwable.getMessage());
        }
        if (writeCause) {
            writeException(throwable.getCause());
        }
        ElasticsearchException.writeStackTraces(throwable, this);
    }
}

Example 25 with IndexFormatTooOldException

use of org.apache.lucene.index.IndexFormatTooOldException in project elasticsearch by elastic.

the class ChecksumBlobStoreFormat method readBlob.

/**
     * Reads blob with specified name without resolving the blobName using using {@link #blobName} method.
     *
     * @param blobContainer blob container
     * @param blobName blob name
     */
public T readBlob(BlobContainer blobContainer, String blobName) throws IOException {
    try (InputStream inputStream = blobContainer.readBlob(blobName)) {
        ByteArrayOutputStream out = new ByteArrayOutputStream();
        Streams.copy(inputStream, out);
        final byte[] bytes = out.toByteArray();
        final String resourceDesc = "ChecksumBlobStoreFormat.readBlob(blob=\"" + blobName + "\")";
        try (ByteArrayIndexInput indexInput = new ByteArrayIndexInput(resourceDesc, bytes)) {
            CodecUtil.checksumEntireFile(indexInput);
            CodecUtil.checkHeader(indexInput, codec, VERSION, VERSION);
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            BytesReference bytesReference = new BytesArray(bytes, (int) filePointer, (int) contentSize);
            return read(bytesReference);
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}