Examples with CorruptIndexException org.apache.lucene.index.CorruptIndexException used on opensource projects

Example 56 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class SimpleTransLog method replay.

/** Replays ops between start and end location against the provided writer.  Can run concurrently with ongoing operations. */
public void replay(NodeProcess primary, long start, long end) throws IOException {
    try (Connection c = new Connection(primary.tcpPort)) {
        c.out.writeByte(SimplePrimaryNode.CMD_INDEXING);
        byte[] intBuffer = new byte[4];
        ByteBuffer intByteBuffer = ByteBuffer.wrap(intBuffer);
        ByteArrayDataInput in = new ByteArrayDataInput();
        long pos = start;
        while (pos < end) {
            intByteBuffer.position(0);
            intByteBuffer.limit(4);
            readBytesFromChannel(pos, intByteBuffer);
            pos += 4;
            int len = ((intBuffer[0] & 0xff) << 24) | (intBuffer[1] & 0xff) << 16 | (intBuffer[2] & 0xff) << 8 | (intBuffer[3] & 0xff);
            byte[] bytes = new byte[len];
            readBytesFromChannel(pos, ByteBuffer.wrap(bytes));
            pos += len;
            in.reset(bytes);
            byte op = in.readByte();
            //System.out.println("xlog: replay op=" + op);
            switch(op) {
                case 0:
                    // We replay add as update:
                    replayAddDocument(c, primary, in);
                    break;
                case 1:
                    // We replay add as update:
                    replayAddDocument(c, primary, in);
                    break;
                case 2:
                    replayDeleteDocuments(c, primary, in);
                    break;
                default:
                    throw new CorruptIndexException("invalid operation " + op, in);
            }
        }
        assert pos == end;
        //System.out.println("xlog: done replay");
        c.out.writeByte(SimplePrimaryNode.CMD_INDEXING_DONE);
        c.flush();
        //System.out.println("xlog: done flush");
        c.in.readByte();
    //System.out.println("xlog: done readByte");
    }
}

Example 57 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class TestCodecUtil method testTruncatedFileThrowsCorruptIndexException.

public void testTruncatedFileThrowsCorruptIndexException() throws IOException {
    RAMFile file = new RAMFile();
    IndexOutput output = new RAMOutputStream(file, false);
    output.close();
    IndexInput input = new RAMInputStream("file", file);
    CorruptIndexException e = expectThrows(CorruptIndexException.class, () -> CodecUtil.checksumEntireFile(input));
    assertEquals("misplaced codec footer (file truncated?): length=0 but footerLength==16 (resource=RAMInputStream(name=file))", e.getMessage());
    e = expectThrows(CorruptIndexException.class, () -> CodecUtil.retrieveChecksum(input));
    assertEquals("misplaced codec footer (file truncated?): length=0 but footerLength==16 (resource=RAMInputStream(name=file))", e.getMessage());
}

Example 58 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class Lucene62SegmentInfoFormat method read.

@Override
public SegmentInfo read(Directory dir, String segment, byte[] segmentID, IOContext context) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segment, "", Lucene62SegmentInfoFormat.SI_EXTENSION);
    try (ChecksumIndexInput input = dir.openChecksumInput(fileName, context)) {
        Throwable priorE = null;
        SegmentInfo si = null;
        try {
            int format = CodecUtil.checkIndexHeader(input, Lucene62SegmentInfoFormat.CODEC_NAME, Lucene62SegmentInfoFormat.VERSION_START, Lucene62SegmentInfoFormat.VERSION_CURRENT, segmentID, "");
            final Version version = Version.fromBits(input.readInt(), input.readInt(), input.readInt());
            final int docCount = input.readInt();
            if (docCount < 0) {
                throw new CorruptIndexException("invalid docCount: " + docCount, input);
            }
            final boolean isCompoundFile = input.readByte() == SegmentInfo.YES;
            final Map<String, String> diagnostics = input.readMapOfStrings();
            final Set<String> files = input.readSetOfStrings();
            final Map<String, String> attributes = input.readMapOfStrings();
            int numSortFields = input.readVInt();
            Sort indexSort;
            if (numSortFields > 0) {
                SortField[] sortFields = new SortField[numSortFields];
                for (int i = 0; i < numSortFields; i++) {
                    String fieldName = input.readString();
                    int sortTypeID = input.readVInt();
                    SortField.Type sortType;
                    SortedSetSelector.Type sortedSetSelector = null;
                    SortedNumericSelector.Type sortedNumericSelector = null;
                    switch(sortTypeID) {
                        case 0:
                            sortType = SortField.Type.STRING;
                            break;
                        case 1:
                            sortType = SortField.Type.LONG;
                            break;
                        case 2:
                            sortType = SortField.Type.INT;
                            break;
                        case 3:
                            sortType = SortField.Type.DOUBLE;
                            break;
                        case 4:
                            sortType = SortField.Type.FLOAT;
                            break;
                        case 5:
                            sortType = SortField.Type.STRING;
                            byte selector = input.readByte();
                            if (selector == 0) {
                                sortedSetSelector = SortedSetSelector.Type.MIN;
                            } else if (selector == 1) {
                                sortedSetSelector = SortedSetSelector.Type.MAX;
                            } else if (selector == 2) {
                                sortedSetSelector = SortedSetSelector.Type.MIDDLE_MIN;
                            } else if (selector == 3) {
                                sortedSetSelector = SortedSetSelector.Type.MIDDLE_MAX;
                            } else {
                                throw new CorruptIndexException("invalid index SortedSetSelector ID: " + selector, input);
                            }
                            break;
                        case 6:
                            byte type = input.readByte();
                            if (type == 0) {
                                sortType = SortField.Type.LONG;
                            } else if (type == 1) {
                                sortType = SortField.Type.INT;
                            } else if (type == 2) {
                                sortType = SortField.Type.DOUBLE;
                            } else if (type == 3) {
                                sortType = SortField.Type.FLOAT;
                            } else {
                                throw new CorruptIndexException("invalid index SortedNumericSortField type ID: " + type, input);
                            }
                            byte numericSelector = input.readByte();
                            if (numericSelector == 0) {
                                sortedNumericSelector = SortedNumericSelector.Type.MIN;
                            } else if (numericSelector == 1) {
                                sortedNumericSelector = SortedNumericSelector.Type.MAX;
                            } else {
                                throw new CorruptIndexException("invalid index SortedNumericSelector ID: " + numericSelector, input);
                            }
                            break;
                        default:
                            throw new CorruptIndexException("invalid index sort field type ID: " + sortTypeID, input);
                    }
                    byte b = input.readByte();
                    boolean reverse;
                    if (b == 0) {
                        reverse = true;
                    } else if (b == 1) {
                        reverse = false;
                    } else {
                        throw new CorruptIndexException("invalid index sort reverse: " + b, input);
                    }
                    if (sortedSetSelector != null) {
                        sortFields[i] = new SortedSetSortField(fieldName, reverse, sortedSetSelector);
                    } else if (sortedNumericSelector != null) {
                        sortFields[i] = new SortedNumericSortField(fieldName, sortType, reverse, sortedNumericSelector);
                    } else {
                        sortFields[i] = new SortField(fieldName, sortType, reverse);
                    }
                    Object missingValue;
                    b = input.readByte();
                    if (b == 0) {
                        missingValue = null;
                    } else {
                        switch(sortType) {
                            case STRING:
                                if (b == 1) {
                                    missingValue = SortField.STRING_LAST;
                                } else if (b == 2) {
                                    missingValue = SortField.STRING_FIRST;
                                } else {
                                    throw new CorruptIndexException("invalid missing value flag: " + b, input);
                                }
                                break;
                            case LONG:
                                if (b != 1) {
                                    throw new CorruptIndexException("invalid missing value flag: " + b, input);
                                }
                                missingValue = input.readLong();
                                break;
                            case INT:
                                if (b != 1) {
                                    throw new CorruptIndexException("invalid missing value flag: " + b, input);
                                }
                                missingValue = input.readInt();
                                break;
                            case DOUBLE:
                                if (b != 1) {
                                    throw new CorruptIndexException("invalid missing value flag: " + b, input);
                                }
                                missingValue = Double.longBitsToDouble(input.readLong());
                                break;
                            case FLOAT:
                                if (b != 1) {
                                    throw new CorruptIndexException("invalid missing value flag: " + b, input);
                                }
                                missingValue = Float.intBitsToFloat(input.readInt());
                                break;
                            default:
                                throw new AssertionError("unhandled sortType=" + sortType);
                        }
                    }
                    if (missingValue != null) {
                        sortFields[i].setMissingValue(missingValue);
                    }
                }
                indexSort = new Sort(sortFields);
            } else if (numSortFields < 0) {
                throw new CorruptIndexException("invalid index sort field count: " + numSortFields, input);
            } else {
                indexSort = null;
            }
            si = new SegmentInfo(dir, version, null, segment, docCount, isCompoundFile, null, diagnostics, segmentID, attributes, indexSort);
            si.setFiles(files);
        } catch (Throwable exception) {
            priorE = exception;
        } finally {
            CodecUtil.checkFooter(input, priorE);
        }
        return si;
    }
}

Example 59 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class Lucene50LiveDocsFormat method writeLiveDocs.

@Override
public void writeLiveDocs(MutableBits bits, Directory dir, SegmentCommitInfo info, int newDelCount, IOContext context) throws IOException {
    long gen = info.getNextDelGen();
    String name = IndexFileNames.fileNameFromGeneration(info.info.name, EXTENSION, gen);
    FixedBitSet fbs = (FixedBitSet) bits;
    if (fbs.length() - fbs.cardinality() != info.getDelCount() + newDelCount) {
        throw new CorruptIndexException("bits.deleted=" + (fbs.length() - fbs.cardinality()) + " info.delcount=" + info.getDelCount() + " newdelcount=" + newDelCount, name);
    }
    long[] data = fbs.getBits();
    try (IndexOutput output = dir.createOutput(name, context)) {
        CodecUtil.writeIndexHeader(output, CODEC_NAME, VERSION_CURRENT, info.info.getId(), Long.toString(gen, Character.MAX_RADIX));
        for (int i = 0; i < data.length; i++) {
            output.writeLong(data[i]);
        }
        CodecUtil.writeFooter(output);
    }
}

Example 60 with CorruptIndexException

use of org.apache.lucene.index.CorruptIndexException in project lucene-solr by apache.

the class Lucene60FieldInfosFormat method read.

@Override
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
    try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) {
        Throwable priorE = null;
        FieldInfo[] infos = null;
        try {
            CodecUtil.checkIndexHeader(input, Lucene60FieldInfosFormat.CODEC_NAME, Lucene60FieldInfosFormat.FORMAT_START, Lucene60FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
            //read in the size
            final int size = input.readVInt();
            infos = new FieldInfo[size];
            // previous field's attribute map, we share when possible:
            Map<String, String> lastAttributes = Collections.emptyMap();
            for (int i = 0; i < size; i++) {
                String name = input.readString();
                final int fieldNumber = input.readVInt();
                if (fieldNumber < 0) {
                    throw new CorruptIndexException("invalid field number for field: " + name + ", fieldNumber=" + fieldNumber, input);
                }
                byte bits = input.readByte();
                boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;
                boolean omitNorms = (bits & OMIT_NORMS) != 0;
                boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
                final IndexOptions indexOptions = getIndexOptions(input, input.readByte());
                // DV Types are packed in one byte
                final DocValuesType docValuesType = getDocValuesType(input, input.readByte());
                final long dvGen = input.readLong();
                Map<String, String> attributes = input.readMapOfStrings();
                // just use the last field's map if its the same
                if (attributes.equals(lastAttributes)) {
                    attributes = lastAttributes;
                }
                lastAttributes = attributes;
                int pointDimensionCount = input.readVInt();
                int pointNumBytes;
                if (pointDimensionCount != 0) {
                    pointNumBytes = input.readVInt();
                } else {
                    pointNumBytes = 0;
                }
                try {
                    infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, attributes, pointDimensionCount, pointNumBytes);
                    infos[i].checkConsistency();
                } catch (IllegalStateException e) {
                    throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
                }
            }
        } catch (Throwable exception) {
            priorE = exception;
        } finally {
            CodecUtil.checkFooter(input, priorE);
        }
        return new FieldInfos(infos);
    }
}