Examples with IndexFormatTooNewException org.apache.lucene.index.IndexFormatTooNewException used on opensource projects

Example 6 with IndexFormatTooNewException

use of org.apache.lucene.index.IndexFormatTooNewException in project elasticsearch by elastic.

the class StreamInput method readException.

public <T extends Exception> T readException() throws IOException {
    if (readBoolean()) {
        int key = readVInt();
        switch(key) {
            case 0:
                final int ord = readVInt();
                return (T) ElasticsearchException.readException(this, ord);
            case 1:
                String msg1 = readOptionalString();
                String resource1 = readOptionalString();
                return (T) readStackTrace(new CorruptIndexException(msg1, resource1, readException()), this);
            case 2:
                String resource2 = readOptionalString();
                int version2 = readInt();
                int minVersion2 = readInt();
                int maxVersion2 = readInt();
                return (T) readStackTrace(new IndexFormatTooNewException(resource2, version2, minVersion2, maxVersion2), this);
            case 3:
                String resource3 = readOptionalString();
                if (readBoolean()) {
                    int version3 = readInt();
                    int minVersion3 = readInt();
                    int maxVersion3 = readInt();
                    return (T) readStackTrace(new IndexFormatTooOldException(resource3, version3, minVersion3, maxVersion3), this);
                } else {
                    String version3 = readOptionalString();
                    return (T) readStackTrace(new IndexFormatTooOldException(resource3, version3), this);
                }
            case 4:
                return (T) readStackTrace(new NullPointerException(readOptionalString()), this);
            case 5:
                return (T) readStackTrace(new NumberFormatException(readOptionalString()), this);
            case 6:
                return (T) readStackTrace(new IllegalArgumentException(readOptionalString(), readException()), this);
            case 7:
                return (T) readStackTrace(new AlreadyClosedException(readOptionalString(), readException()), this);
            case 8:
                return (T) readStackTrace(new EOFException(readOptionalString()), this);
            case 9:
                return (T) readStackTrace(new SecurityException(readOptionalString(), readException()), this);
            case 10:
                return (T) readStackTrace(new StringIndexOutOfBoundsException(readOptionalString()), this);
            case 11:
                return (T) readStackTrace(new ArrayIndexOutOfBoundsException(readOptionalString()), this);
            case 12:
                return (T) readStackTrace(new FileNotFoundException(readOptionalString()), this);
            case 13:
                final int subclass = readVInt();
                final String file = readOptionalString();
                final String other = readOptionalString();
                final String reason = readOptionalString();
                // skip the msg - it's composed from file, other and reason
                readOptionalString();
                final Exception exception;
                switch(subclass) {
                    case 0:
                        exception = new NoSuchFileException(file, other, reason);
                        break;
                    case 1:
                        exception = new NotDirectoryException(file);
                        break;
                    case 2:
                        exception = new DirectoryNotEmptyException(file);
                        break;
                    case 3:
                        exception = new AtomicMoveNotSupportedException(file, other, reason);
                        break;
                    case 4:
                        exception = new FileAlreadyExistsException(file, other, reason);
                        break;
                    case 5:
                        exception = new AccessDeniedException(file, other, reason);
                        break;
                    case 6:
                        exception = new FileSystemLoopException(file);
                        break;
                    case 7:
                        exception = new FileSystemException(file, other, reason);
                        break;
                    default:
                        throw new IllegalStateException("unknown FileSystemException with index " + subclass);
                }
                return (T) readStackTrace(exception, this);
            case 14:
                return (T) readStackTrace(new IllegalStateException(readOptionalString(), readException()), this);
            case 15:
                return (T) readStackTrace(new LockObtainFailedException(readOptionalString(), readException()), this);
            case 16:
                return (T) readStackTrace(new InterruptedException(readOptionalString()), this);
            case 17:
                return (T) readStackTrace(new IOException(readOptionalString(), readException()), this);
            default:
                assert false : "no such exception for id: " + key;
        }
    }
    return null;
}

Example 7 with IndexFormatTooNewException

use of org.apache.lucene.index.IndexFormatTooNewException in project elasticsearch by elastic.

the class RecoverySourceHandler method phase1.

/**
     * Perform phase1 of the recovery operations. Once this {@link IndexCommit}
     * snapshot has been performed no commit operations (files being fsync'd)
     * are effectively allowed on this index until all recovery phases are done
     * <p>
     * Phase1 examines the segment files on the target node and copies over the
     * segments that are missing. Only segments that have the same size and
     * checksum can be reused
     */
public void phase1(final IndexCommit snapshot, final Translog.View translogView) {
    cancellableThreads.checkForCancel();
    // Total size of segment files that are recovered
    long totalSize = 0;
    // Total size of segment files that were able to be re-used
    long existingTotalSize = 0;
    final Store store = shard.store();
    store.incRef();
    try {
        StopWatch stopWatch = new StopWatch().start();
        final Store.MetadataSnapshot recoverySourceMetadata;
        try {
            recoverySourceMetadata = store.getMetadata(snapshot);
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            shard.failShard("recovery", ex);
            throw ex;
        }
        for (String name : snapshot.getFileNames()) {
            final StoreFileMetaData md = recoverySourceMetadata.get(name);
            if (md == null) {
                logger.info("Snapshot differs from actual index for file: {} meta: {}", name, recoverySourceMetadata.asMap());
                throw new CorruptIndexException("Snapshot differs from actual index - maybe index was removed metadata has " + recoverySourceMetadata.asMap().size() + " files", name);
            }
        }
        // Generate a "diff" of all the identical, different, and missing
        // segment files on the target node, using the existing files on
        // the source node
        String recoverySourceSyncId = recoverySourceMetadata.getSyncId();
        String recoveryTargetSyncId = request.metadataSnapshot().getSyncId();
        final boolean recoverWithSyncId = recoverySourceSyncId != null && recoverySourceSyncId.equals(recoveryTargetSyncId);
        if (recoverWithSyncId) {
            final long numDocsTarget = request.metadataSnapshot().getNumDocs();
            final long numDocsSource = recoverySourceMetadata.getNumDocs();
            if (numDocsTarget != numDocsSource) {
                throw new IllegalStateException("try to recover " + request.shardId() + " from primary shard with sync id but number " + "of docs differ: " + numDocsSource + " (" + request.sourceNode().getName() + ", primary) vs " + numDocsTarget + "(" + request.targetNode().getName() + ")");
            }
            // we shortcut recovery here because we have nothing to copy. but we must still start the engine on the target.
            // so we don't return here
            logger.trace("skipping [phase1]- identical sync id [{}] found on both source and target", recoverySourceSyncId);
        } else {
            final Store.RecoveryDiff diff = recoverySourceMetadata.recoveryDiff(request.metadataSnapshot());
            for (StoreFileMetaData md : diff.identical) {
                response.phase1ExistingFileNames.add(md.name());
                response.phase1ExistingFileSizes.add(md.length());
                existingTotalSize += md.length();
                if (logger.isTraceEnabled()) {
                    logger.trace("recovery [phase1]: not recovering [{}], exist in local store and has checksum [{}]," + " size [{}]", md.name(), md.checksum(), md.length());
                }
                totalSize += md.length();
            }
            List<StoreFileMetaData> phase1Files = new ArrayList<>(diff.different.size() + diff.missing.size());
            phase1Files.addAll(diff.different);
            phase1Files.addAll(diff.missing);
            for (StoreFileMetaData md : phase1Files) {
                if (request.metadataSnapshot().asMap().containsKey(md.name())) {
                    logger.trace("recovery [phase1]: recovering [{}], exists in local store, but is different: remote [{}], local [{}]", md.name(), request.metadataSnapshot().asMap().get(md.name()), md);
                } else {
                    logger.trace("recovery [phase1]: recovering [{}], does not exist in remote", md.name());
                }
                response.phase1FileNames.add(md.name());
                response.phase1FileSizes.add(md.length());
                totalSize += md.length();
            }
            response.phase1TotalSize = totalSize;
            response.phase1ExistingTotalSize = existingTotalSize;
            logger.trace("recovery [phase1]: recovering_files [{}] with total_size [{}], reusing_files [{}] with total_size [{}]", response.phase1FileNames.size(), new ByteSizeValue(totalSize), response.phase1ExistingFileNames.size(), new ByteSizeValue(existingTotalSize));
            cancellableThreads.execute(() -> recoveryTarget.receiveFileInfo(response.phase1FileNames, response.phase1FileSizes, response.phase1ExistingFileNames, response.phase1ExistingFileSizes, translogView.totalOperations()));
            // How many bytes we've copied since we last called RateLimiter.pause
            final Function<StoreFileMetaData, OutputStream> outputStreamFactories = md -> new BufferedOutputStream(new RecoveryOutputStream(md, translogView), chunkSizeInBytes);
            sendFiles(store, phase1Files.toArray(new StoreFileMetaData[phase1Files.size()]), outputStreamFactories);
            // are deleted
            try {
                cancellableThreads.executeIO(() -> recoveryTarget.cleanFiles(translogView.totalOperations(), recoverySourceMetadata));
            } catch (RemoteTransportException | IOException targetException) {
                final IOException corruptIndexException;
                //   - maybe due to old segments without checksums or length only checks
                if ((corruptIndexException = ExceptionsHelper.unwrapCorruption(targetException)) != null) {
                    try {
                        final Store.MetadataSnapshot recoverySourceMetadata1 = store.getMetadata(snapshot);
                        StoreFileMetaData[] metadata = StreamSupport.stream(recoverySourceMetadata1.spliterator(), false).toArray(size -> new StoreFileMetaData[size]);
                        ArrayUtil.timSort(metadata, (o1, o2) -> {
                            // check small files first
                            return Long.compare(o1.length(), o2.length());
                        });
                        for (StoreFileMetaData md : metadata) {
                            cancellableThreads.checkForCancel();
                            logger.debug("checking integrity for file {} after remove corruption exception", md);
                            if (store.checkIntegrityNoException(md) == false) {
                                // we are corrupted on the primary -- fail!
                                shard.failShard("recovery", corruptIndexException);
                                logger.warn("Corrupted file detected {} checksum mismatch", md);
                                throw corruptIndexException;
                            }
                        }
                    } catch (IOException ex) {
                        targetException.addSuppressed(ex);
                        throw targetException;
                    }
                    // corruption has happened on the way to replica
                    RemoteTransportException exception = new RemoteTransportException("File corruption occurred on recovery but " + "checksums are ok", null);
                    exception.addSuppressed(targetException);
                    logger.warn((org.apache.logging.log4j.util.Supplier<?>) () -> new ParameterizedMessage("{} Remote file corruption during finalization of recovery on node {}. local checksum OK", shard.shardId(), request.targetNode()), corruptIndexException);
                    throw exception;
                } else {
                    throw targetException;
                }
            }
        }
        logger.trace("recovery [phase1]: took [{}]", stopWatch.totalTime());
        response.phase1Time = stopWatch.totalTime().millis();
    } catch (Exception e) {
        throw new RecoverFilesRecoveryException(request.shardId(), response.phase1FileNames.size(), new ByteSizeValue(totalSize), e);
    } finally {
        store.decRef();
    }
}

Example 8 with IndexFormatTooNewException

use of org.apache.lucene.index.IndexFormatTooNewException in project elasticsearch by elastic.

the class RecoveryTarget method cleanFiles.

@Override
public void cleanFiles(int totalTranslogOps, Store.MetadataSnapshot sourceMetaData) throws IOException {
    state().getTranslog().totalOperations(totalTranslogOps);
    // first, we go and move files that were created with the recovery id suffix to
    // the actual names, its ok if we have a corrupted index here, since we have replicas
    // to recover from in case of a full cluster shutdown just when this code executes...
    renameAllTempFiles();
    final Store store = store();
    try {
        store.cleanupAndVerify("recovery CleanFilesRequestHandler", sourceMetaData);
    } catch (CorruptIndexException | IndexFormatTooNewException | IndexFormatTooOldException ex) {
        // its content on disk if possible.
        try {
            try {
                store.removeCorruptionMarker();
            } finally {
                // clean up and delete all files
                Lucene.cleanLuceneIndex(store.directory());
            }
        } catch (Exception e) {
            logger.debug("Failed to clean lucene index", e);
            ex.addSuppressed(e);
        }
        RecoveryFailedException rfe = new RecoveryFailedException(state(), "failed to clean after recovery", ex);
        fail(rfe, true);
        throw rfe;
    } catch (Exception ex) {
        RecoveryFailedException rfe = new RecoveryFailedException(state(), "failed to clean after recovery", ex);
        fail(rfe, true);
        throw rfe;
    }
}

Example 9 with IndexFormatTooNewException

use of org.apache.lucene.index.IndexFormatTooNewException in project elasticsearch by elastic.

the class MetaDataStateFormat method read.

/**
     * Reads the state from a given file and compares the expected version against the actual version of
     * the state.
     */
public final T read(NamedXContentRegistry namedXContentRegistry, Path file) throws IOException {
    try (Directory dir = newDirectory(file.getParent())) {
        try (IndexInput indexInput = dir.openInput(file.getFileName().toString(), IOContext.DEFAULT)) {
            // We checksum the entire file before we even go and parse it. If it's corrupted we barf right here.
            CodecUtil.checksumEntireFile(indexInput);
            final int fileVersion = CodecUtil.checkHeader(indexInput, STATE_FILE_CODEC, MIN_COMPATIBLE_STATE_FILE_VERSION, STATE_FILE_VERSION);
            final XContentType xContentType = XContentType.values()[indexInput.readInt()];
            if (fileVersion == STATE_FILE_VERSION_ES_2X_AND_BELOW) {
                // format version 0, wrote a version that always came from the content state file and was never used
                // version currently unused
                indexInput.readLong();
            }
            long filePointer = indexInput.getFilePointer();
            long contentSize = indexInput.length() - CodecUtil.footerLength() - filePointer;
            try (IndexInput slice = indexInput.slice("state_xcontent", filePointer, contentSize)) {
                try (XContentParser parser = XContentFactory.xContent(xContentType).createParser(namedXContentRegistry, new InputStreamIndexInput(slice, contentSize))) {
                    return fromXContent(parser);
                }
            }
        } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException ex) {
            // we trick this into a dedicated exception with the original stacktrace
            throw new CorruptStateException(ex);
        }
    }
}

Example 10 with IndexFormatTooNewException

use of org.apache.lucene.index.IndexFormatTooNewException in project elasticsearch by elastic.

the class TranslogReader method open.

/**
     * Given a file channel, opens a {@link TranslogReader}, taking care of checking and validating the file header.
     *
     * @param channel the translog file channel
     * @param path the path to the translog
     * @param checkpoint the translog checkpoint
     * @param translogUUID the tranlog UUID
     * @return a new TranslogReader
     * @throws IOException if any of the file operations resulted in an I/O exception
     */
public static TranslogReader open(final FileChannel channel, final Path path, final Checkpoint checkpoint, final String translogUUID) throws IOException {
    try {
        // don't close
        InputStreamStreamInput headerStream = new InputStreamStreamInput(java.nio.channels.Channels.newInputStream(channel));
        // Lucene's CodecUtil writes a magic number of 0x3FD76C17 with the
        // header, in binary this looks like:
        //
        // binary: 0011 1111 1101 0111 0110 1100 0001 0111
        // hex   :    3    f    d    7    6    c    1    7
        //
        // With version 0 of the translog, the first byte is the
        // Operation.Type, which will always be between 0-4, so we know if
        // we grab the first byte, it can be:
        // 0x3f => Lucene's magic number, so we can assume it's version 1 or later
        // 0x00 => version 0 of the translog
        //
        // otherwise the first byte of the translog is corrupted and we
        // should bail
        byte b1 = headerStream.readByte();
        if (b1 == LUCENE_CODEC_HEADER_BYTE) {
            // Read 3 more bytes, meaning a whole integer has been read
            byte b2 = headerStream.readByte();
            byte b3 = headerStream.readByte();
            byte b4 = headerStream.readByte();
            // Convert the 4 bytes that were read into an integer
            int header = ((b1 & 0xFF) << 24) + ((b2 & 0xFF) << 16) + ((b3 & 0xFF) << 8) + ((b4 & 0xFF) << 0);
            // byte separately
            if (header != CodecUtil.CODEC_MAGIC) {
                throw new TranslogCorruptedException("translog looks like version 1 or later, but has corrupted header. path:" + path);
            }
            // Confirm the rest of the header using CodecUtil, extracting
            // the translog version
            int version = CodecUtil.checkHeaderNoMagic(new InputStreamDataInput(headerStream), TranslogWriter.TRANSLOG_CODEC, 1, Integer.MAX_VALUE);
            switch(version) {
                case TranslogWriter.VERSION_CHECKSUMS:
                    throw new IllegalStateException("pre-2.0 translog found [" + path + "]");
                case TranslogWriter.VERSION_CHECKPOINTS:
                    assert path.getFileName().toString().endsWith(Translog.TRANSLOG_FILE_SUFFIX) : "new file ends with old suffix: " + path;
                    assert checkpoint.numOps >= 0 : "expected at least 0 operatin but got: " + checkpoint.numOps;
                    assert checkpoint.offset <= channel.size() : "checkpoint is inconsistent with channel length: " + channel.size() + " " + checkpoint;
                    int len = headerStream.readInt();
                    if (len > channel.size()) {
                        throw new TranslogCorruptedException("uuid length can't be larger than the translog");
                    }
                    BytesRef ref = new BytesRef(len);
                    ref.length = len;
                    headerStream.read(ref.bytes, ref.offset, ref.length);
                    BytesRef uuidBytes = new BytesRef(translogUUID);
                    if (uuidBytes.bytesEquals(ref) == false) {
                        throw new TranslogCorruptedException("expected shard UUID " + uuidBytes + " but got: " + ref + " this translog file belongs to a different translog. path:" + path);
                    }
                    final long firstOperationOffset = ref.length + CodecUtil.headerLength(TranslogWriter.TRANSLOG_CODEC) + Integer.BYTES;
                    return new TranslogReader(checkpoint, channel, path, firstOperationOffset);
                default:
                    throw new TranslogCorruptedException("No known translog stream version: " + version + " path:" + path);
            }
        } else if (b1 == UNVERSIONED_TRANSLOG_HEADER_BYTE) {
            throw new IllegalStateException("pre-1.4 translog found [" + path + "]");
        } else {
            throw new TranslogCorruptedException("Invalid first byte in translog file, got: " + Long.toHexString(b1) + ", expected 0x00 or 0x3f. path:" + path);
        }
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
        throw new TranslogCorruptedException("Translog header corrupted. path:" + path, e);
    }
}