Examples with FieldInfos org.apache.lucene.index.FieldInfos used on opensource projects

Example 1 with FieldInfos

use of org.apache.lucene.index.FieldInfos in project lucene-solr by apache.

the class SimpleTextFieldInfosFormat method read.

@Override
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext iocontext) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, FIELD_INFOS_EXTENSION);
    ChecksumIndexInput input = directory.openChecksumInput(fileName, iocontext);
    BytesRefBuilder scratch = new BytesRefBuilder();
    boolean success = false;
    try {
        SimpleTextUtil.readLine(input, scratch);
        assert StringHelper.startsWith(scratch.get(), NUMFIELDS);
        final int size = Integer.parseInt(readString(NUMFIELDS.length, scratch));
        FieldInfo[] infos = new FieldInfo[size];
        for (int i = 0; i < size; i++) {
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NAME);
            String name = readString(NAME.length, scratch);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NUMBER);
            int fieldNumber = Integer.parseInt(readString(NUMBER.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), INDEXOPTIONS);
            String s = readString(INDEXOPTIONS.length, scratch);
            final IndexOptions indexOptions = IndexOptions.valueOf(s);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), STORETV);
            boolean storeTermVector = Boolean.parseBoolean(readString(STORETV.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), PAYLOADS);
            boolean storePayloads = Boolean.parseBoolean(readString(PAYLOADS.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NORMS);
            boolean omitNorms = !Boolean.parseBoolean(readString(NORMS.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DOCVALUES);
            String dvType = readString(DOCVALUES.length, scratch);
            final DocValuesType docValuesType = docValuesType(dvType);
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DOCVALUES_GEN);
            final long dvGen = Long.parseLong(readString(DOCVALUES_GEN.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), NUM_ATTS);
            int numAtts = Integer.parseInt(readString(NUM_ATTS.length, scratch));
            Map<String, String> atts = new HashMap<>();
            for (int j = 0; j < numAtts; j++) {
                SimpleTextUtil.readLine(input, scratch);
                assert StringHelper.startsWith(scratch.get(), ATT_KEY);
                String key = readString(ATT_KEY.length, scratch);
                SimpleTextUtil.readLine(input, scratch);
                assert StringHelper.startsWith(scratch.get(), ATT_VALUE);
                String value = readString(ATT_VALUE.length, scratch);
                atts.put(key, value);
            }
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DIM_COUNT);
            int dimensionalCount = Integer.parseInt(readString(DIM_COUNT.length, scratch));
            SimpleTextUtil.readLine(input, scratch);
            assert StringHelper.startsWith(scratch.get(), DIM_NUM_BYTES);
            int dimensionalNumBytes = Integer.parseInt(readString(DIM_NUM_BYTES.length, scratch));
            infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, Collections.unmodifiableMap(atts), dimensionalCount, dimensionalNumBytes);
        }
        SimpleTextUtil.checkFooter(input);
        FieldInfos fieldInfos = new FieldInfos(infos);
        success = true;
        return fieldInfos;
    } finally {
        if (success) {
            input.close();
        } else {
            IOUtils.closeWhileHandlingException(input);
        }
    }
}

Example 2 with FieldInfos

use of org.apache.lucene.index.FieldInfos in project lucene-solr by apache.

the class Lucene50FieldInfosFormat method read.

@Override
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
    try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) {
        Throwable priorE = null;
        FieldInfo[] infos = null;
        try {
            CodecUtil.checkIndexHeader(input, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_START, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
            //read in the size
            final int size = input.readVInt();
            infos = new FieldInfo[size];
            // previous field's attribute map, we share when possible:
            Map<String, String> lastAttributes = Collections.emptyMap();
            for (int i = 0; i < size; i++) {
                String name = input.readString();
                final int fieldNumber = input.readVInt();
                if (fieldNumber < 0) {
                    throw new CorruptIndexException("invalid field number for field: " + name + ", fieldNumber=" + fieldNumber, input);
                }
                byte bits = input.readByte();
                boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;
                boolean omitNorms = (bits & OMIT_NORMS) != 0;
                boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
                final IndexOptions indexOptions = getIndexOptions(input, input.readByte());
                // DV Types are packed in one byte
                final DocValuesType docValuesType = getDocValuesType(input, input.readByte());
                final long dvGen = input.readLong();
                Map<String, String> attributes = input.readMapOfStrings();
                // just use the last field's map if its the same
                if (attributes.equals(lastAttributes)) {
                    attributes = lastAttributes;
                }
                lastAttributes = attributes;
                try {
                    infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, attributes, 0, 0);
                    infos[i].checkConsistency();
                } catch (IllegalStateException e) {
                    throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
                }
            }
        } catch (Throwable exception) {
            priorE = exception;
        } finally {
            CodecUtil.checkFooter(input, priorE);
        }
        return new FieldInfos(infos);
    }
}

Example 3 with FieldInfos

use of org.apache.lucene.index.FieldInfos in project lucene-solr by apache.

the class UninvertDocValuesMergePolicyTest method testNonIndexedFieldDoesNonFail.

// When an non-indexed field gets merged, it exhibit the old behavior
// The field will be merged, docvalues headers updated, but no docvalues for this field
public void testNonIndexedFieldDoesNonFail() throws Exception {
    // Remove Indexed from fieldType
    removeIndexFrom(h, TEST_FIELD);
    assertU(adoc(ID_FIELD, String.valueOf(1), TEST_FIELD, String.valueOf(1)));
    assertU(commit());
    addDocValuesTo(h, TEST_FIELD);
    assertU(adoc(ID_FIELD, String.valueOf(2), TEST_FIELD, String.valueOf(2)));
    assertU(commit());
    assertU(optimize("maxSegments", "1"));
    withNewRawReader(h, topReader -> {
        assertEquals(2, topReader.numDocs());
        assertEquals(1, topReader.leaves().size());
        final FieldInfos infos = MultiFields.getMergedFieldInfos(topReader);
        assertEquals(DocValuesType.SORTED, infos.fieldInfo(TEST_FIELD).getDocValuesType());
        for (LeafReaderContext ctx : topReader.leaves()) {
            LeafReader r = ctx.reader();
            SortedDocValues docvalues = r.getSortedDocValues(TEST_FIELD);
            for (int i = 0; i < r.numDocs(); ++i) {
                Document doc = r.document(i);
                String v = doc.getField(TEST_FIELD).stringValue();
                String id = doc.getField(ID_FIELD).stringValue();
                assertEquals(DocValuesType.SORTED, r.getFieldInfos().fieldInfo(TEST_FIELD).getDocValuesType());
                assertEquals(DocValuesType.NONE, r.getFieldInfos().fieldInfo(ID_FIELD).getDocValuesType());
                if (id.equals("2")) {
                    assertTrue(docvalues.advanceExact(i));
                    assertEquals(v, docvalues.binaryValue().utf8ToString());
                } else {
                    assertFalse(docvalues.advanceExact(i));
                }
            }
        }
    });
}

Example 4 with FieldInfos

use of org.apache.lucene.index.FieldInfos in project lucene-solr by apache.

the class DocValuesTest method testDocValues.

@Test
public void testDocValues() throws IOException {
    assertU(adoc("id", "1"));
    assertU(commit());
    try (SolrCore core = h.getCoreInc()) {
        final RefCounted<SolrIndexSearcher> searcherRef = core.openNewSearcher(true, true);
        final SolrIndexSearcher searcher = searcherRef.get();
        try {
            final LeafReader reader = searcher.getSlowAtomicReader();
            assertEquals(1, reader.numDocs());
            final FieldInfos infos = reader.getFieldInfos();
            assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("floatdv").getDocValuesType());
            assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("intdv").getDocValuesType());
            assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("doubledv").getDocValuesType());
            assertEquals(DocValuesType.NUMERIC, infos.fieldInfo("longdv").getDocValuesType());
            assertEquals(DocValuesType.SORTED, infos.fieldInfo("stringdv").getDocValuesType());
            assertEquals(DocValuesType.SORTED, infos.fieldInfo("booldv").getDocValuesType());
            NumericDocValues dvs = reader.getNumericDocValues("floatdv");
            assertEquals(0, dvs.nextDoc());
            assertEquals((long) Float.floatToIntBits(1), dvs.longValue());
            dvs = reader.getNumericDocValues("intdv");
            assertEquals(0, dvs.nextDoc());
            assertEquals(2L, dvs.longValue());
            dvs = reader.getNumericDocValues("doubledv");
            assertEquals(0, dvs.nextDoc());
            assertEquals(Double.doubleToLongBits(3), dvs.longValue());
            dvs = reader.getNumericDocValues("longdv");
            assertEquals(0, dvs.nextDoc());
            assertEquals(4L, dvs.longValue());
            SortedDocValues sdv = reader.getSortedDocValues("stringdv");
            assertEquals(0, sdv.nextDoc());
            assertEquals("solr", sdv.binaryValue().utf8ToString());
            sdv = reader.getSortedDocValues("booldv");
            assertEquals(0, sdv.nextDoc());
            assertEquals("T", sdv.binaryValue().utf8ToString());
            final IndexSchema schema = core.getLatestSchema();
            final SchemaField floatDv = schema.getField("floatdv");
            final SchemaField intDv = schema.getField("intdv");
            final SchemaField doubleDv = schema.getField("doubledv");
            final SchemaField longDv = schema.getField("longdv");
            final SchemaField boolDv = schema.getField("booldv");
            FunctionValues values = floatDv.getType().getValueSource(floatDv, null).getValues(null, searcher.getSlowAtomicReader().leaves().get(0));
            assertEquals(1f, values.floatVal(0), 0f);
            assertEquals(1f, values.objectVal(0));
            values = intDv.getType().getValueSource(intDv, null).getValues(null, searcher.getSlowAtomicReader().leaves().get(0));
            assertEquals(2, values.intVal(0));
            assertEquals(2, values.objectVal(0));
            values = doubleDv.getType().getValueSource(doubleDv, null).getValues(null, searcher.getSlowAtomicReader().leaves().get(0));
            assertEquals(3d, values.doubleVal(0), 0d);
            assertEquals(3d, values.objectVal(0));
            values = longDv.getType().getValueSource(longDv, null).getValues(null, searcher.getSlowAtomicReader().leaves().get(0));
            assertEquals(4L, values.longVal(0));
            assertEquals(4L, values.objectVal(0));
            values = boolDv.getType().getValueSource(boolDv, null).getValues(null, searcher.getSlowAtomicReader().leaves().get(0));
            assertEquals("true", values.strVal(0));
            assertEquals(true, values.objectVal(0));
            // check reversibility of created fields
            tstToObj(schema.getField("floatdv"), -1.5f);
            tstToObj(schema.getField("floatdvs"), -1.5f);
            tstToObj(schema.getField("doubledv"), -1.5d);
            tstToObj(schema.getField("doubledvs"), -1.5d);
            tstToObj(schema.getField("intdv"), -7);
            tstToObj(schema.getField("intdvs"), -7);
            tstToObj(schema.getField("longdv"), -11L);
            tstToObj(schema.getField("longdvs"), -11L);
            tstToObj(schema.getField("datedv"), new Date(1000));
            tstToObj(schema.getField("datedvs"), new Date(1000));
            tstToObj(schema.getField("stringdv"), "foo");
            tstToObj(schema.getField("stringdvs"), "foo");
            tstToObj(schema.getField("booldv"), true);
            tstToObj(schema.getField("booldvs"), true);
        } finally {
            searcherRef.decref();
        }
    }
}

Example 5 with FieldInfos

use of org.apache.lucene.index.FieldInfos in project lucene-solr by apache.

the class UnifiedHighlighter method getFieldInfo.

/**
   * Called by the default implementation of {@link #getOffsetSource(String)}.
   * If there is no searcher then we simply always return null.
   */
protected FieldInfo getFieldInfo(String field) {
    if (searcher == null) {
        return null;
    }
    // Need thread-safety for lazy-init but lets avoid 'synchronized' by using double-check locking idiom
    // note: it's volatile; read once
    FieldInfos fieldInfos = this.fieldInfos;
    if (fieldInfos == null) {
        synchronized (this) {
            fieldInfos = this.fieldInfos;
            if (fieldInfos == null) {
                fieldInfos = MultiFields.getMergedFieldInfos(searcher.getIndexReader());
                this.fieldInfos = fieldInfos;
            }
        }
    }
    return fieldInfos.fieldInfo(field);
}