Examples with CorruptIndexException org.apache.lucene.index.CorruptIndexException used on opensource projects

Example 36 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class SimpleTextPointsReader method checkIntegrity.

@Override
public void checkIntegrity() throws IOException {
    BytesRefBuilder scratch = new BytesRefBuilder();
    IndexInput clone = dataIn.clone();
    clone.seek(0);
    // checksum is fixed-width encoded with 20 bytes, plus 1 byte for newline (the space is included in SimpleTextUtil.CHECKSUM):
    long footerStartPos = dataIn.length() - (SimpleTextUtil.CHECKSUM.length + 21);
    ChecksumIndexInput input = new BufferedChecksumIndexInput(clone);
    while (true) {
        SimpleTextUtil.readLine(input, scratch);
        if (input.getFilePointer() >= footerStartPos) {
            // Make sure we landed at precisely the right location:
            if (input.getFilePointer() != footerStartPos) {
                throw new CorruptIndexException("SimpleText failure: footer does not start at expected position current=" + input.getFilePointer() + " vs expected=" + footerStartPos, input);
            }
            SimpleTextUtil.checkFooter(input);
            break;
        }
    }
}

Example 37 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class SimpleTextUtil method checkFooter.

public static void checkFooter(ChecksumIndexInput input) throws IOException {
    BytesRefBuilder scratch = new BytesRefBuilder();
    String expectedChecksum = String.format(Locale.ROOT, "%020d", input.getChecksum());
    readLine(input, scratch);
    if (StringHelper.startsWith(scratch.get(), CHECKSUM) == false) {
        throw new CorruptIndexException("SimpleText failure: expected checksum line but got " + scratch.get().utf8ToString(), input);
    }
    String actualChecksum = new BytesRef(scratch.bytes(), CHECKSUM.length, scratch.length() - CHECKSUM.length).utf8ToString();
    if (!expectedChecksum.equals(actualChecksum)) {
        throw new CorruptIndexException("SimpleText checksum failure: " + actualChecksum + " != " + expectedChecksum, input);
    }
    if (input.length() != input.getFilePointer()) {
        throw new CorruptIndexException("Unexpected stuff at the end of file, please be careful with your text editor!", input);
    }
}

Example 38 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class SimpleTextCompoundFormat method getCompoundReader.

@Override
public Directory getCompoundReader(Directory dir, SegmentInfo si, IOContext context) throws IOException {
    String dataFile = IndexFileNames.segmentFileName(si.name, "", DATA_EXTENSION);
    final IndexInput in = dir.openInput(dataFile, context);
    BytesRefBuilder scratch = new BytesRefBuilder();
    // first get to TOC:
    DecimalFormat df = new DecimalFormat(OFFSETPATTERN, DecimalFormatSymbols.getInstance(Locale.ROOT));
    long pos = in.length() - TABLEPOS.length - OFFSETPATTERN.length() - 1;
    in.seek(pos);
    SimpleTextUtil.readLine(in, scratch);
    assert StringHelper.startsWith(scratch.get(), TABLEPOS);
    long tablePos = -1;
    try {
        tablePos = df.parse(stripPrefix(scratch, TABLEPOS)).longValue();
    } catch (ParseException e) {
        throw new CorruptIndexException("can't parse CFS trailer, got: " + scratch.get().utf8ToString(), in);
    }
    // seek to TOC and read it
    in.seek(tablePos);
    SimpleTextUtil.readLine(in, scratch);
    assert StringHelper.startsWith(scratch.get(), TABLE);
    int numEntries = Integer.parseInt(stripPrefix(scratch, TABLE));
    final String[] fileNames = new String[numEntries];
    final long[] startOffsets = new long[numEntries];
    final long[] endOffsets = new long[numEntries];
    for (int i = 0; i < numEntries; i++) {
        SimpleTextUtil.readLine(in, scratch);
        assert StringHelper.startsWith(scratch.get(), TABLENAME);
        fileNames[i] = si.name + IndexFileNames.stripSegmentName(stripPrefix(scratch, TABLENAME));
        if (i > 0) {
            // files must be unique and in sorted order
            assert fileNames[i].compareTo(fileNames[i - 1]) > 0;
        }
        SimpleTextUtil.readLine(in, scratch);
        assert StringHelper.startsWith(scratch.get(), TABLESTART);
        startOffsets[i] = Long.parseLong(stripPrefix(scratch, TABLESTART));
        SimpleTextUtil.readLine(in, scratch);
        assert StringHelper.startsWith(scratch.get(), TABLEEND);
        endOffsets[i] = Long.parseLong(stripPrefix(scratch, TABLEEND));
    }
    return new Directory() {

        private int getIndex(String name) throws IOException {
            int index = Arrays.binarySearch(fileNames, name);
            if (index < 0) {
                throw new FileNotFoundException("No sub-file found (fileName=" + name + " files: " + Arrays.toString(fileNames) + ")");
            }
            return index;
        }

        @Override
        public String[] listAll() throws IOException {
            ensureOpen();
            return fileNames.clone();
        }

        @Override
        public long fileLength(String name) throws IOException {
            ensureOpen();
            int index = getIndex(name);
            return endOffsets[index] - startOffsets[index];
        }

        @Override
        public IndexInput openInput(String name, IOContext context) throws IOException {
            ensureOpen();
            int index = getIndex(name);
            return in.slice(name, startOffsets[index], endOffsets[index] - startOffsets[index]);
        }

        @Override
        public void close() throws IOException {
            in.close();
        }

        // write methods: disabled
        @Override
        public IndexOutput createOutput(String name, IOContext context) {
            throw new UnsupportedOperationException();
        }

        @Override
        public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void sync(Collection<String> names) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void deleteFile(String name) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void rename(String source, String dest) {
            throw new UnsupportedOperationException();
        }

        @Override
        public void syncMetaData() {
            throw new UnsupportedOperationException();
        }

        @Override
        public Lock obtainLock(String name) {
            throw new UnsupportedOperationException();
        }
    };
}

Example 39 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project bigbluebutton by bigbluebutton.

the class Index method startIndex.

public void startIndex(String uid) {
    try {
        IndexReader.unlock(FSDirectory.getDirectory(ConfigHandler.indexPath));
        if (logger.isInfoEnabled()) {
            logger.info("index file path " + ConfigHandler.indexPath);
        }
        reader = IndexReader.open(ConfigHandler.indexPath);
        TermEnum uidIter = reader.terms(new Term("uid"));
        while (uidIter.term() != null) {
            if (uid.equalsIgnoreCase(uidIter.term().text())) {
                reader.deleteDocuments(uidIter.term());
            }
            uidIter.next();
        }
        reader.close();
    } catch (CorruptIndexException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (LockObtainFailedException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
    try {
        writer = new IndexWriter(ConfigHandler.indexPath, new StandardAnalyzer(), new IndexWriter.MaxFieldLength(1000000));
    } catch (CorruptIndexException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (LockObtainFailedException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    } catch (IOException e) {
        // TODO Auto-generated catch block
        e.printStackTrace();
    }
}

Example 40 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project elasticsearch by elastic.

the class TranslogReader method open.

/**
     * Given a file channel, opens a {@link TranslogReader}, taking care of checking and validating the file header.
     *
     * @param channel the translog file channel
     * @param path the path to the translog
     * @param checkpoint the translog checkpoint
     * @param translogUUID the tranlog UUID
     * @return a new TranslogReader
     * @throws IOException if any of the file operations resulted in an I/O exception
     */
public static TranslogReader open(final FileChannel channel, final Path path, final Checkpoint checkpoint, final String translogUUID) throws IOException {
    try {
        // don't close
        InputStreamStreamInput headerStream = new InputStreamStreamInput(java.nio.channels.Channels.newInputStream(channel));
        // Lucene's CodecUtil writes a magic number of 0x3FD76C17 with the
        // header, in binary this looks like:
        //
        // binary: 0011 1111 1101 0111 0110 1100 0001 0111
        // hex   :    3    f    d    7    6    c    1    7
        //
        // With version 0 of the translog, the first byte is the
        // Operation.Type, which will always be between 0-4, so we know if
        // we grab the first byte, it can be:
        // 0x3f => Lucene's magic number, so we can assume it's version 1 or later
        // 0x00 => version 0 of the translog
        //
        // otherwise the first byte of the translog is corrupted and we
        // should bail
        byte b1 = headerStream.readByte();
        if (b1 == LUCENE_CODEC_HEADER_BYTE) {
            // Read 3 more bytes, meaning a whole integer has been read
            byte b2 = headerStream.readByte();
            byte b3 = headerStream.readByte();
            byte b4 = headerStream.readByte();
            // Convert the 4 bytes that were read into an integer
            int header = ((b1 & 0xFF) << 24) + ((b2 & 0xFF) << 16) + ((b3 & 0xFF) << 8) + ((b4 & 0xFF) << 0);
            // byte separately
            if (header != CodecUtil.CODEC_MAGIC) {
                throw new TranslogCorruptedException("translog looks like version 1 or later, but has corrupted header. path:" + path);
            }
            // Confirm the rest of the header using CodecUtil, extracting
            // the translog version
            int version = CodecUtil.checkHeaderNoMagic(new InputStreamDataInput(headerStream), TranslogWriter.TRANSLOG_CODEC, 1, Integer.MAX_VALUE);
            switch(version) {
                case TranslogWriter.VERSION_CHECKSUMS:
                    throw new IllegalStateException("pre-2.0 translog found [" + path + "]");
                case TranslogWriter.VERSION_CHECKPOINTS:
                    assert path.getFileName().toString().endsWith(Translog.TRANSLOG_FILE_SUFFIX) : "new file ends with old suffix: " + path;
                    assert checkpoint.numOps >= 0 : "expected at least 0 operatin but got: " + checkpoint.numOps;
                    assert checkpoint.offset <= channel.size() : "checkpoint is inconsistent with channel length: " + channel.size() + " " + checkpoint;
                    int len = headerStream.readInt();
                    if (len > channel.size()) {
                        throw new TranslogCorruptedException("uuid length can't be larger than the translog");
                    }
                    BytesRef ref = new BytesRef(len);
                    ref.length = len;
                    headerStream.read(ref.bytes, ref.offset, ref.length);
                    BytesRef uuidBytes = new BytesRef(translogUUID);
                    if (uuidBytes.bytesEquals(ref) == false) {
                        throw new TranslogCorruptedException("expected shard UUID " + uuidBytes + " but got: " + ref + " this translog file belongs to a different translog. path:" + path);
                    }
                    final long firstOperationOffset = ref.length + CodecUtil.headerLength(TranslogWriter.TRANSLOG_CODEC) + Integer.BYTES;
                    return new TranslogReader(checkpoint, channel, path, firstOperationOffset);
                default:
                    throw new TranslogCorruptedException("No known translog stream version: " + version + " path:" + path);
            }
        } else if (b1 == UNVERSIONED_TRANSLOG_HEADER_BYTE) {
            throw new IllegalStateException("pre-1.4 translog found [" + path + "]");
        } else {
            throw new TranslogCorruptedException("Invalid first byte in translog file, got: " + Long.toHexString(b1) + ", expected 0x00 or 0x3f. path:" + path);
        }
    } catch (CorruptIndexException | IndexFormatTooOldException | IndexFormatTooNewException e) {
        throw new TranslogCorruptedException("Translog header corrupted. path:" + path, e);
    }
}