Examples with CorruptIndexException org.apache.lucene.index.CorruptIndexException used on opensource projects

Example 76 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class Lucene60FieldInfosFormat method read.

@Override
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
    try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) {
        Throwable priorE = null;
        FieldInfo[] infos = null;
        try {
            CodecUtil.checkIndexHeader(input, Lucene60FieldInfosFormat.CODEC_NAME, Lucene60FieldInfosFormat.FORMAT_START, Lucene60FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
            //read in the size
            final int size = input.readVInt();
            infos = new FieldInfo[size];
            // previous field's attribute map, we share when possible:
            Map<String, String> lastAttributes = Collections.emptyMap();
            for (int i = 0; i < size; i++) {
                String name = input.readString();
                final int fieldNumber = input.readVInt();
                if (fieldNumber < 0) {
                    throw new CorruptIndexException("invalid field number for field: " + name + ", fieldNumber=" + fieldNumber, input);
                }
                byte bits = input.readByte();
                boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;
                boolean omitNorms = (bits & OMIT_NORMS) != 0;
                boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
                final IndexOptions indexOptions = getIndexOptions(input, input.readByte());
                // DV Types are packed in one byte
                final DocValuesType docValuesType = getDocValuesType(input, input.readByte());
                final long dvGen = input.readLong();
                Map<String, String> attributes = input.readMapOfStrings();
                // just use the last field's map if its the same
                if (attributes.equals(lastAttributes)) {
                    attributes = lastAttributes;
                }
                lastAttributes = attributes;
                int pointDimensionCount = input.readVInt();
                int pointNumBytes;
                if (pointDimensionCount != 0) {
                    pointNumBytes = input.readVInt();
                } else {
                    pointNumBytes = 0;
                }
                try {
                    infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, attributes, pointDimensionCount, pointNumBytes);
                    infos[i].checkConsistency();
                } catch (IllegalStateException e) {
                    throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
                }
            }
        } catch (Throwable exception) {
            priorE = exception;
        } finally {
            CodecUtil.checkFooter(input, priorE);
        }
        return new FieldInfos(infos);
    }
}

Example 77 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class Lucene70SegmentInfoFormat method read.

@Override
public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene70SegmentInfoFormat.SI_EXTENSION);
    try (ChecksumIndexInput input = dir.openChecksumInput(fileName, context)) {
        Throwable priorE = null;
        SegmentInfo si = null;
        try {
            int format = CodecUtil.checkIndexHeader(input, Lucene70SegmentInfoFormat.CODEC_NAME, Lucene70SegmentInfoFormat.VERSION_START, Lucene70SegmentInfoFormat.VERSION_CURRENT, segmentID, "");
            final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
            byte hasMinVersion = input.readByte();
            final Version minVersion;
            switch(hasMinVersion) {
                case 0:
                    minVersion = null;
                    break;
                case 1:
                    minVersion = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
                    break;
                default:
                    throw new CorruptIndexException("Illegal boolean value " + hasMinVersion, input);
            }
            final int docCount = input.readInt();
            if (docCount < 0) {
                throw new CorruptIndexException("invalid docCount: " + docCount, input);
            }
            final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
            final Map<String, String> diagnostics = input.readMapOfStrings();
            final Set<String> files = input.readSetOfStrings();
            final Map<String, String> attributes = input.readMapOfStrings();
            int numSortFields = input.readVInt();
            Sort indexSort;
            if (numSortFields > 0) {
                SortField[] sortFields = new SortField[numSortFields];
                for (int i = 0; i < numSortFields; i++) {
                    String fieldName = input.readString();
                    int sortTypeID = input.readVInt();
                    SortField.Type sortType;
                    SortedSetSelector.Type sortedSetSelector = null;
                    SortedNumericSelector.Type sortedNumericSelector = null;
                    switch(sortTypeID) {
                        case 0:
                            sortType = SortField.Type.STRING;
                            break;
                        case 1:
                            sortType = SortField.Type.LONG;
                            break;
                        case 2:
                            sortType = SortField.Type.INT;
                            break;
                        case 3:
                            sortType = SortField.Type.DOUBLE;
                            break;
                        case 4:
                            sortType = SortField.Type.FLOAT;
                            break;
                        case 5:
                            sortType = SortField.Type.STRING;
                            byte selector = input.readByte();
                            if (selector == 0) {
                                sortedSetSelector = SortedSetSelector.Type.MIN;
                            } else if (selector == 1) {
                                sortedSetSelector = SortedSetSelector.Type.MAX;
                            } else if (selector == 2) {
                                sortedSetSelector = SortedSetSelector.Type.MIDDLE_MIN;
                            } else if (selector == 3) {
                                sortedSetSelector = SortedSetSelector.Type.MIDDLE_MAX;
                            } else {
                                throw new CorruptIndexException("invalid index SortedSetSelector ID: " + selector, input);
                            }
                            break;
                        case 6:
                            byte type = input.readByte();
                            if (type == 0) {
                                sortType = SortField.Type.LONG;
                            } else if (type == 1) {
                                sortType = SortField.Type.INT;
                            } else if (type == 2) {
                                sortType = SortField.Type.DOUBLE;
                            } else if (type == 3) {
                                sortType = SortField.Type.FLOAT;
                            } else {
                                throw new CorruptIndexException("invalid index SortedNumericSortField type ID: " + type, input);
                            }
                            byte numericSelector = input.readByte();
                            if (numericSelector == 0) {
                                sortedNumericSelector = SortedNumericSelector.Type.MIN;
                            } else if (numericSelector == 1) {
                                sortedNumericSelector = SortedNumericSelector.Type.MAX;
                            } else {
                                throw new CorruptIndexException("invalid index SortedNumericSelector ID: " + numericSelector, input);
                            }
                            break;
                        default:
                            throw new CorruptIndexException("invalid index sort field type ID: " + sortTypeID, input);
                    }
                    byte b = input.readByte();
                    boolean reverse;
                    if (b == 0) {
                        reverse = true;
                    } else if (b == 1) {
                        reverse = false;
                    } else {
                        throw new CorruptIndexException("invalid index sort reverse: " + b, input);
                    }
                    if (sortedSetSelector != null) {
                        sortFields[i] = new SortedSetSortField(fieldName, reverse, sortedSetSelector);
                    } else if (sortedNumericSelector != null) {
                        sortFields[i] = new SortedNumericSortField(fieldName, sortType, reverse, sortedNumericSelector);
                    } else {
                        sortFields[i] = new SortField(fieldName, sortType, reverse);
                    }
                    Object missingValue;
                    b = input.readByte();
                    if (b == 0) {
                        missingValue = null;
                    } else {
                        switch(sortType) {
                            case STRING:
                                if (b == 1) {
                                    missingValue = SortField.STRING_LAST;
                                } else if (b == 2) {
                                    missingValue = SortField.STRING_FIRST;
                                } else {
                                    throw new CorruptIndexException("invalid missing value flag: " + b, input);
                                }
                                break;
                            case LONG:
                                if (b != 1) {
                                    throw new CorruptIndexException("invalid missing value flag: " + b, input);
                                }
                                missingValue = input.readLong();
                                break;
                            case INT:
                                if (b != 1) {
                                    throw new CorruptIndexException("invalid missing value flag: " + b, input);
                                }
                                missingValue = input.readInt();
                                break;
                            case DOUBLE:
                                if (b != 1) {
                                    throw new CorruptIndexException("invalid missing value flag: " + b, input);
                                }
                                missingValue = Double.longBitsToDouble(input.readLong());
                                break;
                            case FLOAT:
                                if (b != 1) {
                                    throw new CorruptIndexException("invalid missing value flag: " + b, input);
                                }
                                missingValue = Float.intBitsToFloat(input.readInt());
                                break;
                            default:
                                throw new AssertionError("unhandled sortType=" + sortType);
                        }
                    }
                    if (missingValue != null) {
                        sortFields[i].setMissingValue(missingValue);
                    }
                }
                indexSort = new Sort(sortFields);
            } else if (numSortFields < 0) {
                throw new CorruptIndexException("invalid index sort field count: " + numSortFields, input);
            } else {
                indexSort = null;
            }
            si = new SegmentInfo(dir, version, minVersion, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, indexSort);
            si.setFiles(files);
        } catch (Throwable exception) {
            priorE = exception;
        } finally {
            CodecUtil.checkFooter(input, priorE);
        }
        return si;
    }
}

Example 78 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class Lucene50CompoundReader method readEntries.

/** Helper method that reads CFS entries from an input stream */
private Map<String, FileEntry> readEntries(byte[] segmentID, Directory dir, String entriesFileName) throws IOException {
    Map<String, FileEntry> mapping = null;
    try (ChecksumIndexInput entriesStream = dir.openChecksumInput(entriesFileName, IOContext.READONCE)) {
        Throwable priorE = null;
        try {
            version = CodecUtil.checkIndexHeader(entriesStream, Lucene50CompoundFormat.ENTRY_CODEC, Lucene50CompoundFormat.VERSION_START, Lucene50CompoundFormat.VERSION_CURRENT, segmentID, "");
            final int numEntries = entriesStream.readVInt();
            mapping = new HashMap<>(numEntries);
            for (int i = 0; i < numEntries; i++) {
                final FileEntry fileEntry = new FileEntry();
                final String id = entriesStream.readString();
                FileEntry previous = mapping.put(id, fileEntry);
                if (previous != null) {
                    throw new CorruptIndexException("Duplicate cfs entry id=" + id + " in CFS ", entriesStream);
                }
                fileEntry.offset = entriesStream.readLong();
                fileEntry.length = entriesStream.readLong();
            }
        } catch (Throwable exception) {
            priorE = exception;
        } finally {
            CodecUtil.checkFooter(entriesStream, priorE);
        }
    }
    return Collections.unmodifiableMap(mapping);
}

Example 79 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class TestBKD method testBitFlippedOnPartition1.

/** Make sure corruption on an input sort file is caught, even if BKDWriter doesn't get angry */
public void testBitFlippedOnPartition1() throws Exception {
    // Generate fixed data set:
    int numDocs = atLeast(10000);
    int numBytesPerDim = 4;
    int numDims = 3;
    byte[][][] docValues = new byte[numDocs][][];
    byte counter = 0;
    for (int docID = 0; docID < numDocs; docID++) {
        byte[][] values = new byte[numDims][];
        for (int dim = 0; dim < numDims; dim++) {
            values[dim] = new byte[numBytesPerDim];
            for (int i = 0; i < values[dim].length; i++) {
                values[dim][i] = counter;
                counter++;
            }
        }
        docValues[docID] = values;
    }
    try (Directory dir0 = newMockDirectory()) {
        Directory dir = new FilterDirectory(dir0) {

            boolean corrupted;

            @Override
            public IndexOutput createTempOutput(String prefix, String suffix, IOContext context) throws IOException {
                IndexOutput out = in.createTempOutput(prefix, suffix, context);
                if (corrupted == false && prefix.equals("_0_bkd1") && suffix.equals("sort")) {
                    corrupted = true;
                    return new CorruptingIndexOutput(dir0, 22, out);
                } else {
                    return out;
                }
            }
        };
        CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> {
            verify(dir, docValues, null, numDims, numBytesPerDim, 50, 0.1);
        });
        assertTrue(e.getMessage().contains("checksum failed (hardware problem?)"));
    }
}

Example 80 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project elki by elki-project.

the class LuceneDatabase method initialize.

@Override
public void initialize() {
    try {
        reader = IndexReader.open(directory);
        ids = DBIDUtil.generateStaticDBIDRange(reader.maxDoc());
        // ID relation:
        idrep = new DBIDView(ids);
        relations.add(idrep);
        getHierarchy().add(this, idrep);
        // Documents relation:
        docrep = new LuceneDocumentRelation(ids, reader);
        relations.add(docrep);
        getHierarchy().add(this, docrep);
        eventManager.fireObjectsInserted(ids);
    } catch (CorruptIndexException e) {
        throw new AbortException("Index is corrupt.", e);
    } catch (IOException e) {
        throw new AbortException("I/O error reading index.", e);
    }
}