Examples with IndexOptions org.apache.lucene.index.IndexOptions used on opensource projects

Example 1 with IndexOptions

use of org.apache.lucene.index.IndexOptions in project lucene-solr by apache.

the class TestBlockPostingsFormat2 method newDocument.

private Document newDocument() {
    Document doc = new Document();
    for (IndexOptions option : IndexOptions.values()) {
        if (option == IndexOptions.NONE) {
            continue;
        }
        FieldType ft = new FieldType(TextField.TYPE_NOT_STORED);
        // turn on tvs for a cross-check, since we rely upon checkindex in this test (for now)
        ft.setStoreTermVectors(true);
        ft.setStoreTermVectorOffsets(true);
        ft.setStoreTermVectorPositions(true);
        ft.setStoreTermVectorPayloads(true);
        ft.setIndexOptions(option);
        doc.add(new Field(option.toString(), "", ft));
    }
    return doc;
}

Example 2 with IndexOptions

use of org.apache.lucene.index.IndexOptions in project lucene-solr by apache.

the class SimpleTextFieldInfosFormat method write.

@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, FIELD_INFOS_EXTENSION);
    IndexOutput out = directory.createOutput(fileName, context);
    BytesRefBuilder scratch = new BytesRefBuilder();
    boolean success = false;
    try {
        SimpleTextUtil.write(out, NUMFIELDS);
        SimpleTextUtil.write(out, Integer.toString(infos.size()), scratch);
        SimpleTextUtil.writeNewline(out);
        for (FieldInfo fi : infos) {
            SimpleTextUtil.write(out, NAME);
            SimpleTextUtil.write(out, fi.name, scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, NUMBER);
            SimpleTextUtil.write(out, Integer.toString(fi.number), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, INDEXOPTIONS);
            IndexOptions indexOptions = fi.getIndexOptions();
            assert indexOptions.compareTo(IndexOptions.DOCS_AND_FREQS_AND_POSITIONS) >= 0 || !fi.hasPayloads();
            SimpleTextUtil.write(out, indexOptions.toString(), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, STORETV);
            SimpleTextUtil.write(out, Boolean.toString(fi.hasVectors()), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, PAYLOADS);
            SimpleTextUtil.write(out, Boolean.toString(fi.hasPayloads()), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, NORMS);
            SimpleTextUtil.write(out, Boolean.toString(!fi.omitsNorms()), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, DOCVALUES);
            SimpleTextUtil.write(out, getDocValuesType(fi.getDocValuesType()), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, DOCVALUES_GEN);
            SimpleTextUtil.write(out, Long.toString(fi.getDocValuesGen()), scratch);
            SimpleTextUtil.writeNewline(out);
            Map<String, String> atts = fi.attributes();
            int numAtts = atts == null ? 0 : atts.size();
            SimpleTextUtil.write(out, NUM_ATTS);
            SimpleTextUtil.write(out, Integer.toString(numAtts), scratch);
            SimpleTextUtil.writeNewline(out);
            if (numAtts > 0) {
                for (Map.Entry<String, String> entry : atts.entrySet()) {
                    SimpleTextUtil.write(out, ATT_KEY);
                    SimpleTextUtil.write(out, entry.getKey(), scratch);
                    SimpleTextUtil.writeNewline(out);
                    SimpleTextUtil.write(out, ATT_VALUE);
                    SimpleTextUtil.write(out, entry.getValue(), scratch);
                    SimpleTextUtil.writeNewline(out);
                }
            }
            SimpleTextUtil.write(out, DIM_COUNT);
            SimpleTextUtil.write(out, Integer.toString(fi.getPointDimensionCount()), scratch);
            SimpleTextUtil.writeNewline(out);
            SimpleTextUtil.write(out, DIM_NUM_BYTES);
            SimpleTextUtil.write(out, Integer.toString(fi.getPointNumBytes()), scratch);
            SimpleTextUtil.writeNewline(out);
        }
        SimpleTextUtil.writeChecksum(out, scratch);
        success = true;
    } finally {
        if (success) {
            out.close();
        } else {
            IOUtils.closeWhileHandlingException(out);
        }
    }
}

Example 3 with IndexOptions

use of org.apache.lucene.index.IndexOptions in project lucene-solr by apache.

the class SimpleTextFieldInfosFormat method read.

@Override
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, FIELD_INFOS_EXTENSION);
    ChecksumIndexInput input = directory.openChecksumInput(fileName, iocontext);
    BytesRefBuilder scratch = new BytesRefBuilder();
    boolean success = false;
    try {
        SimpleTextUtil.readLine(input, scratch);
        assert StringHelper.startsWith(scratch.get(), NUMFIELDS);
        final int size = Integer.parseInt(readString(NUMFIELDS.length, scratch));
        FieldInfo[] infos = new FieldInfo[size];
        for (int i = 0; i < size; i++) {
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NAME);
            String name = readString(NAME.length, scratch);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NUMBER);
            int fieldNumber = Integer.parseInt(readString(NUMBER.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), INDEXOPTIONS);
            String s = readString(INDEXOPTIONS.length, scratch);
            final IndexOptions indexOptions = IndexOptions.valueOf(s);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), STORETV);
            boolean storeTermVector = Boolean.parseBoolean(readString(STORETV.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), PAYLOADS);
            boolean storePayloads = Boolean.parseBoolean(readString(PAYLOADS.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NORMS);
            boolean omitNorms = !Boolean.parseBoolean(readString(NORMS.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DOCVALUES);
            String dvType = readString(DOCVALUES.length, scratch);
            final DocValuesType docValuesType = docValuesType(dvType);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DOCVALUES_GEN);
            final long dvGen = Long.parseLong(readString(DOCVALUES_GEN.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NUM_ATTS);
            int numAtts = Integer.parseInt(readString(NUM_ATTS.length, scratch));
            Map<String, String> atts = new HashMap<>();
            for (int j = 0; j < numAtts; j++) {
                SimpleTextUtil.readLine(input, scratch);
                assert StringHelper.startsWith(scratch.get(), ATT_KEY);
                String key = readString(ATT_KEY.length, scratch);
                SimpleTextUtil.readLine(input, scratch);
                assert StringHelper.startsWith(scratch.get(), ATT_VALUE);
                String value = readString(ATT_VALUE.length, scratch);
                atts.put(key, value);
            }
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DIM_COUNT);
            int dimensionalCount = Integer.parseInt(readString(DIM_COUNT.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DIM_NUM_BYTES);
            int dimensionalNumBytes = Integer.parseInt(readString(DIM_NUM_BYTES.length, scratch));
            infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts), dimensionalCount, dimensionalNumBytes);
        }
        SimpleTextUtil.checkFooter(input);
        FieldInfos fieldInfos = new FieldInfos(infos);
        success = true;
        return fieldInfos;
    } finally {
        if (success) {
            input.close();
        } else {
            IOUtils.closeWhileHandlingException(input);
        }
    }
}

Example 4 with IndexOptions

use of org.apache.lucene.index.IndexOptions in project lucene-solr by apache.

the class Lucene50FieldInfosFormat method read.

@Override
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
    try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) {
        Throwable priorE = null;
        FieldInfo[] infos = null;
        try {
            CodecUtil.checkIndexHeader(input, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_START, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
            //read in the size
            final int size = input.readVInt();
            infos = new FieldInfo[size];
            // previous field's attribute map, we share when possible:
            Map<String, String> lastAttributes = Collections.emptyMap();
            for (int i = 0; i < size; i++) {
                String name = input.readString();
                final int fieldNumber = input.readVInt();
                if (fieldNumber < 0) {
                    throw new CorruptIndexException("invalid field number for field: " + name + ", fieldNumber=" + fieldNumber, input);
                }
                byte bits = input.readByte();
                boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;
                boolean omitNorms = (bits & OMIT_NORMS) != 0;
                boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
                final IndexOptions indexOptions = getIndexOptions(input, input.readByte());
                // DV Types are packed in one byte
                final DocValuesType docValuesType = getDocValuesType(input, input.readByte());
                final long dvGen = input.readLong();
                Map<String, String> attributes = input.readMapOfStrings();
                // just use the last field's map if its the same
                if (attributes.equals(lastAttributes)) {
                    attributes = lastAttributes;
                }
                lastAttributes = attributes;
                try {
                    infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, attributes, 0, 0);
                    infos[i].checkConsistency();
                } catch (IllegalStateException e) {
                    throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
                }
            }
        } catch (Throwable exception) {
            priorE = exception;
        } finally {
            CodecUtil.checkFooter(input, priorE);
        }
        return new FieldInfos(infos);
    }
}

Example 5 with IndexOptions

use of org.apache.lucene.index.IndexOptions in project lucene-solr by apache.

the class LukeRequestHandler method getFieldFlags.

/**
   * @return a string representing a IndexableField's flags.  
   */
private static String getFieldFlags(IndexableField f) {
    IndexOptions opts = (f == null) ? null : f.fieldType().indexOptions();
    StringBuilder flags = new StringBuilder();
    flags.append((f != null && f.fieldType().indexOptions() != IndexOptions.NONE) ? FieldFlag.INDEXED.getAbbreviation() : '-');
    flags.append((f != null && f.fieldType().tokenized()) ? FieldFlag.TOKENIZED.getAbbreviation() : '-');
    flags.append((f != null && f.fieldType().stored()) ? FieldFlag.STORED.getAbbreviation() : '-');
    flags.append((f != null && f.fieldType().docValuesType() != DocValuesType.NONE) ? FieldFlag.DOC_VALUES.getAbbreviation() : "-");
    // SchemaField Specific
    flags.append((false) ? FieldFlag.MULTI_VALUED.getAbbreviation() : '-');
    flags.append((f != null && f.fieldType().storeTermVectors()) ? FieldFlag.TERM_VECTOR_STORED.getAbbreviation() : '-');
    flags.append((f != null && f.fieldType().storeTermVectorOffsets()) ? FieldFlag.TERM_VECTOR_OFFSET.getAbbreviation() : '-');
    flags.append((f != null && f.fieldType().storeTermVectorPositions()) ? FieldFlag.TERM_VECTOR_POSITION.getAbbreviation() : '-');
    flags.append((f != null && f.fieldType().storeTermVectorPayloads()) ? FieldFlag.TERM_VECTOR_PAYLOADS.getAbbreviation() : '-');
    flags.append((f != null && f.fieldType().omitNorms()) ? FieldFlag.OMIT_NORMS.getAbbreviation() : '-');
    flags.append((f != null && DOCS == opts) ? FieldFlag.OMIT_TF.getAbbreviation() : '-');
    flags.append((f != null && DOCS_AND_FREQS == opts) ? FieldFlag.OMIT_POSITIONS.getAbbreviation() : '-');
    flags.append((f != null && DOCS_AND_FREQS_AND_POSITIONS_AND_OFFSETS == opts) ? FieldFlag.STORE_OFFSETS_WITH_POSITIONS.getAbbreviation() : '-');
    flags.append((f != null && f.getClass().getSimpleName().equals("LazyField")) ? FieldFlag.LAZY.getAbbreviation() : '-');
    flags.append((f != null && f.binaryValue() != null) ? FieldFlag.BINARY.getAbbreviation() : '-');
    // SchemaField Specific
    flags.append((false) ? FieldFlag.SORT_MISSING_FIRST.getAbbreviation() : '-');
    // SchemaField Specific
    flags.append((false) ? FieldFlag.SORT_MISSING_LAST.getAbbreviation() : '-');
    return flags.toString();
}