Examples with ChecksumIndexInput org.apache.lucene.store.ChecksumIndexInput used on opensource projects

Example 11 with ChecksumIndexInput

use of org.apache.lucene.store.ChecksumIndexInput in project lucene-solr by apache.

the class SimpleTextDocValuesReader method checkIntegrity.

@Override
public void checkIntegrity() throws IOException {
    BytesRefBuilder scratch = new BytesRefBuilder();
    IndexInput clone = data.clone();
    clone.seek(0);
    // checksum is fixed-width encoded with 20 bytes, plus 1 byte for newline (the space is included in SimpleTextUtil.CHECKSUM):
    long footerStartPos = data.length() - (SimpleTextUtil.CHECKSUM.length + 21);
    ChecksumIndexInput input = new BufferedChecksumIndexInput(clone);
    while (true) {
        SimpleTextUtil.readLine(input, scratch);
        if (input.getFilePointer() >= footerStartPos) {
            // Make sure we landed at precisely the right location:
            if (input.getFilePointer() != footerStartPos) {
                throw new CorruptIndexException("SimpleText failure: footer does not start at expected position current=" + input.getFilePointer() + " vs expected=" + footerStartPos, input);
            }
            SimpleTextUtil.checkFooter(input);
            break;
        }
    }
}

Example 12 with ChecksumIndexInput

use of org.apache.lucene.store.ChecksumIndexInput in project lucene-solr by apache.

the class SimpleTextFieldInfosFormat method read.

@Override
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, FIELD_INFOS_EXTENSION);
    ChecksumIndexInput input = directory.openChecksumInput(fileName, iocontext);
    BytesRefBuilder scratch = new BytesRefBuilder();
    boolean success = false;
    try {
        SimpleTextUtil.readLine(input, scratch);
        assert StringHelper.startsWith(scratch.get(), NUMFIELDS);
        final int size = Integer.parseInt(readString(NUMFIELDS.length, scratch));
        FieldInfo[] infos = new FieldInfo[size];
        for (int i = 0; i < size; i++) {
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NAME);
            String name = readString(NAME.length, scratch);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NUMBER);
            int fieldNumber = Integer.parseInt(readString(NUMBER.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), INDEXOPTIONS);
            String s = readString(INDEXOPTIONS.length, scratch);
            final IndexOptions indexOptions = IndexOptions.valueOf(s);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), STORETV);
            boolean storeTermVector = Boolean.parseBoolean(readString(STORETV.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), PAYLOADS);
            boolean storePayloads = Boolean.parseBoolean(readString(PAYLOADS.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NORMS);
            boolean omitNorms = !Boolean.parseBoolean(readString(NORMS.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DOCVALUES);
            String dvType = readString(DOCVALUES.length, scratch);
            final DocValuesType docValuesType = docValuesType(dvType);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DOCVALUES_GEN);
            final long dvGen = Long.parseLong(readString(DOCVALUES_GEN.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NUM_ATTS);
            int numAtts = Integer.parseInt(readString(NUM_ATTS.length, scratch));
            Map<String, String> atts = new HashMap<>();
            for (int j = 0; j < numAtts; j++) {
                SimpleTextUtil.readLine(input, scratch);
                assert StringHelper.startsWith(scratch.get(), ATT_KEY);
                String key = readString(ATT_KEY.length, scratch);
                SimpleTextUtil.readLine(input, scratch);
                assert StringHelper.startsWith(scratch.get(), ATT_VALUE);
                String value = readString(ATT_VALUE.length, scratch);
                atts.put(key, value);
            }
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DIM_COUNT);
            int dimensionalCount = Integer.parseInt(readString(DIM_COUNT.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DIM_NUM_BYTES);
            int dimensionalNumBytes = Integer.parseInt(readString(DIM_NUM_BYTES.length, scratch));
            infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts), dimensionalCount, dimensionalNumBytes);
        }
        SimpleTextUtil.checkFooter(input);
        FieldInfos fieldInfos = new FieldInfos(infos);
        success = true;
        return fieldInfos;
    } finally {
        if (success) {
            input.close();
        } else {
            IOUtils.closeWhileHandlingException(input);
        }
    }
}

Example 13 with ChecksumIndexInput

use of org.apache.lucene.store.ChecksumIndexInput in project lucene-solr by apache.

the class CodecUtil method checksumEntireFile.

/** 
   * Clones the provided input, reads all bytes from the file, and calls {@link #checkFooter} 
   * <p>
   * Note that this method may be slow, as it must process the entire file.
   * If you just need to extract the checksum value, call {@link #retrieveChecksum}.
   */
public static long checksumEntireFile(IndexInput input) throws IOException {
    IndexInput clone = input.clone();
    clone.seek(0);
    ChecksumIndexInput in = new BufferedChecksumIndexInput(clone);
    assert in.getFilePointer() == 0;
    if (in.length() < footerLength()) {
        throw new CorruptIndexException("misplaced codec footer (file truncated?): length=" + in.length() + " but footerLength==" + footerLength(), input);
    }
    in.seek(in.length() - footerLength());
    return checkFooter(in);
}

Example 14 with ChecksumIndexInput

use of org.apache.lucene.store.ChecksumIndexInput in project lucene-solr by apache.

the class SimpleTextLiveDocsFormat method readLiveDocs.

@Override
public Bits readLiveDocs(Directory dir, SegmentCommitInfo info, IOContext context) throws IOException {
    assert info.hasDeletions();
    BytesRefBuilder scratch = new BytesRefBuilder();
    CharsRefBuilder scratchUTF16 = new CharsRefBuilder();
    String fileName = IndexFileNames.fileNameFromGeneration(info.info.name, LIVEDOCS_EXTENSION, info.getDelGen());
    ChecksumIndexInput in = null;
    boolean success = false;
    try {
        in = dir.openChecksumInput(fileName, context);
        SimpleTextUtil.readLine(in, scratch);
        assert StringHelper.startsWith(scratch.get(), SIZE);
        int size = parseIntAt(scratch.get(), SIZE.length, scratchUTF16);
        BitSet bits = new BitSet(size);
        SimpleTextUtil.readLine(in, scratch);
        while (!scratch.get().equals(END)) {
            assert StringHelper.startsWith(scratch.get(), DOC);
            int docid = parseIntAt(scratch.get(), DOC.length, scratchUTF16);
            bits.set(docid);
            SimpleTextUtil.readLine(in, scratch);
        }
        SimpleTextUtil.checkFooter(in);
        success = true;
        return new SimpleTextBits(bits, size);
    } finally {
        if (success) {
            IOUtils.close(in);
        } else {
            IOUtils.closeWhileHandlingException(in);
        }
    }
}

Example 15 with ChecksumIndexInput

use of org.apache.lucene.store.ChecksumIndexInput in project lucene-solr by apache.

the class SimpleTextPointsReader method checkIntegrity.

@Override
public void checkIntegrity() throws IOException {
    BytesRefBuilder scratch = new BytesRefBuilder();
    IndexInput clone = dataIn.clone();
    clone.seek(0);
    // checksum is fixed-width encoded with 20 bytes, plus 1 byte for newline (the space is included in SimpleTextUtil.CHECKSUM):
    long footerStartPos = dataIn.length() - (SimpleTextUtil.CHECKSUM.length + 21);
    ChecksumIndexInput input = new BufferedChecksumIndexInput(clone);
    while (true) {
        SimpleTextUtil.readLine(input, scratch);
        if (input.getFilePointer() >= footerStartPos) {
            // Make sure we landed at precisely the right location:
            if (input.getFilePointer() != footerStartPos) {
                throw new CorruptIndexException("SimpleText failure: footer does not start at expected position current=" + input.getFilePointer() + " vs expected=" + footerStartPos, input);
            }
            SimpleTextUtil.checkFooter(input);
            break;
        }
    }
}