Search in sources :

Example 36 with FieldInfo

use of org.apache.lucene.index.FieldInfo in project lucene-solr by apache.

the class Lucene50FieldInfosFormat method write.

@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
    try (IndexOutput output = directory.createOutput(fileName, context)) {
        CodecUtil.writeIndexHeader(output, Lucene50FieldInfosFormat.CODEC_NAME, Lucene50FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
        output.writeVInt(infos.size());
        for (FieldInfo fi : infos) {
            fi.checkConsistency();
            output.writeString(fi.name);
            output.writeVInt(fi.number);
            byte bits = 0x0;
            if (fi.hasVectors())
                bits |= STORE_TERMVECTOR;
            if (fi.omitsNorms())
                bits |= OMIT_NORMS;
            if (fi.hasPayloads())
                bits |= STORE_PAYLOADS;
            output.writeByte(bits);
            output.writeByte(indexOptionsByte(fi.getIndexOptions()));
            // pack the DV type and hasNorms in one byte
            output.writeByte(docValuesByte(fi.getDocValuesType()));
            output.writeLong(fi.getDocValuesGen());
            output.writeMapOfStrings(fi.attributes());
        }
        CodecUtil.writeFooter(output);
    }
}
Also used : IndexOutput(org.apache.lucene.store.IndexOutput) FieldInfo(org.apache.lucene.index.FieldInfo)

Example 37 with FieldInfo

use of org.apache.lucene.index.FieldInfo in project lucene-solr by apache.

the class Lucene60FieldInfosFormat method write.

@Override
public void write(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, FieldInfos infos, IOContext context) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
    try (IndexOutput output = directory.createOutput(fileName, context)) {
        CodecUtil.writeIndexHeader(output, Lucene60FieldInfosFormat.CODEC_NAME, Lucene60FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
        output.writeVInt(infos.size());
        for (FieldInfo fi : infos) {
            fi.checkConsistency();
            output.writeString(fi.name);
            output.writeVInt(fi.number);
            byte bits = 0x0;
            if (fi.hasVectors())
                bits |= STORE_TERMVECTOR;
            if (fi.omitsNorms())
                bits |= OMIT_NORMS;
            if (fi.hasPayloads())
                bits |= STORE_PAYLOADS;
            output.writeByte(bits);
            output.writeByte(indexOptionsByte(fi.getIndexOptions()));
            // pack the DV type and hasNorms in one byte
            output.writeByte(docValuesByte(fi.getDocValuesType()));
            output.writeLong(fi.getDocValuesGen());
            output.writeMapOfStrings(fi.attributes());
            int pointDimensionCount = fi.getPointDimensionCount();
            output.writeVInt(pointDimensionCount);
            if (pointDimensionCount != 0) {
                output.writeVInt(fi.getPointNumBytes());
            }
        }
        CodecUtil.writeFooter(output);
    }
}
Also used : IndexOutput(org.apache.lucene.store.IndexOutput) FieldInfo(org.apache.lucene.index.FieldInfo)

Example 38 with FieldInfo

use of org.apache.lucene.index.FieldInfo in project lucene-solr by apache.

the class Lucene60FieldInfosFormat method read.

@Override
public FieldInfos read(Directory directory, SegmentInfo segmentInfo, String segmentSuffix, IOContext context) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(segmentInfo.name, segmentSuffix, EXTENSION);
    try (ChecksumIndexInput input = directory.openChecksumInput(fileName, context)) {
        Throwable priorE = null;
        FieldInfo[] infos = null;
        try {
            CodecUtil.checkIndexHeader(input, Lucene60FieldInfosFormat.CODEC_NAME, Lucene60FieldInfosFormat.FORMAT_START, Lucene60FieldInfosFormat.FORMAT_CURRENT, segmentInfo.getId(), segmentSuffix);
            //read in the size
            final int size = input.readVInt();
            infos = new FieldInfo[size];
            // previous field's attribute map, we share when possible:
            Map<String, String> lastAttributes = Collections.emptyMap();
            for (int i = 0; i < size; i++) {
                String name = input.readString();
                final int fieldNumber = input.readVInt();
                if (fieldNumber < 0) {
                    throw new CorruptIndexException("invalid field number for field: " + name + ", fieldNumber=" + fieldNumber, input);
                }
                byte bits = input.readByte();
                boolean storeTermVector = (bits & STORE_TERMVECTOR) != 0;
                boolean omitNorms = (bits & OMIT_NORMS) != 0;
                boolean storePayloads = (bits & STORE_PAYLOADS) != 0;
                final IndexOptions indexOptions = getIndexOptions(input, input.readByte());
                // DV Types are packed in one byte
                final DocValuesType docValuesType = getDocValuesType(input, input.readByte());
                final long dvGen = input.readLong();
                Map<String, String> attributes = input.readMapOfStrings();
                // just use the last field's map if its the same
                if (attributes.equals(lastAttributes)) {
                    attributes = lastAttributes;
                }
                lastAttributes = attributes;
                int pointDimensionCount = input.readVInt();
                int pointNumBytes;
                if (pointDimensionCount != 0) {
                    pointNumBytes = input.readVInt();
                } else {
                    pointNumBytes = 0;
                }
                try {
                    infos[i] = new FieldInfo(name, fieldNumber, storeTermVector, omitNorms, storePayloads, indexOptions, docValuesType, dvGen, attributes, pointDimensionCount, pointNumBytes);
                    infos[i].checkConsistency();
                } catch (IllegalStateException e) {
                    throw new CorruptIndexException("invalid fieldinfo for field: " + name + ", fieldNumber=" + fieldNumber, input, e);
                }
            }
        } catch (Throwable exception) {
            priorE = exception;
        } finally {
            CodecUtil.checkFooter(input, priorE);
        }
        return new FieldInfos(infos);
    }
}
Also used : ChecksumIndexInput(org.apache.lucene.store.ChecksumIndexInput) IndexOptions(org.apache.lucene.index.IndexOptions) CorruptIndexException(org.apache.lucene.index.CorruptIndexException) FieldInfos(org.apache.lucene.index.FieldInfos) DocValuesType(org.apache.lucene.index.DocValuesType) FieldInfo(org.apache.lucene.index.FieldInfo)

Example 39 with FieldInfo

use of org.apache.lucene.index.FieldInfo in project lucene-solr by apache.

the class LatLonPointDistanceQuery method createWeight.

@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
    Rectangle box = Rectangle.fromPointDistance(latitude, longitude, radiusMeters);
    // create bounding box(es) for the distance range
    // these are pre-encoded with LatLonPoint's encoding
    final byte[] minLat = new byte[Integer.BYTES];
    final byte[] maxLat = new byte[Integer.BYTES];
    final byte[] minLon = new byte[Integer.BYTES];
    final byte[] maxLon = new byte[Integer.BYTES];
    // second set of longitude ranges to check (for cross-dateline case)
    final byte[] minLon2 = new byte[Integer.BYTES];
    NumericUtils.intToSortableBytes(encodeLatitude(box.minLat), minLat, 0);
    NumericUtils.intToSortableBytes(encodeLatitude(box.maxLat), maxLat, 0);
    // crosses dateline: split
    if (box.crossesDateline()) {
        // box1
        NumericUtils.intToSortableBytes(Integer.MIN_VALUE, minLon, 0);
        NumericUtils.intToSortableBytes(encodeLongitude(box.maxLon), maxLon, 0);
        // box2
        NumericUtils.intToSortableBytes(encodeLongitude(box.minLon), minLon2, 0);
    } else {
        NumericUtils.intToSortableBytes(encodeLongitude(box.minLon), minLon, 0);
        NumericUtils.intToSortableBytes(encodeLongitude(box.maxLon), maxLon, 0);
        // disable box2
        NumericUtils.intToSortableBytes(Integer.MAX_VALUE, minLon2, 0);
    }
    // compute exact sort key: avoid any asin() computations
    final double sortKey = GeoUtils.distanceQuerySortKey(radiusMeters);
    final double axisLat = Rectangle.axisLat(latitude, radiusMeters);
    return new ConstantScoreWeight(this, boost) {

        final GeoEncodingUtils.DistancePredicate distancePredicate = GeoEncodingUtils.createDistancePredicate(latitude, longitude, radiusMeters);

        @Override
        public Scorer scorer(LeafReaderContext context) throws IOException {
            ScorerSupplier scorerSupplier = scorerSupplier(context);
            if (scorerSupplier == null) {
                return null;
            }
            return scorerSupplier.get(false);
        }

        @Override
        public ScorerSupplier scorerSupplier(LeafReaderContext context) throws IOException {
            LeafReader reader = context.reader();
            PointValues values = reader.getPointValues(field);
            if (values == null) {
                // No docs in this segment had any points fields
                return null;
            }
            FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
            if (fieldInfo == null) {
                // No docs in this segment indexed this field at all
                return null;
            }
            LatLonPoint.checkCompatible(fieldInfo);
            // matching docids
            DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
            final IntersectVisitor visitor = new IntersectVisitor() {

                DocIdSetBuilder.BulkAdder adder;

                @Override
                public void grow(int count) {
                    adder = result.grow(count);
                }

                @Override
                public void visit(int docID) {
                    adder.add(docID);
                }

                @Override
                public void visit(int docID, byte[] packedValue) {
                    // bounding box check
                    if (StringHelper.compare(Integer.BYTES, packedValue, 0, maxLat, 0) > 0 || StringHelper.compare(Integer.BYTES, packedValue, 0, minLat, 0) < 0) {
                        // latitude out of bounding box range
                        return;
                    }
                    if ((StringHelper.compare(Integer.BYTES, packedValue, Integer.BYTES, maxLon, 0) > 0 || StringHelper.compare(Integer.BYTES, packedValue, Integer.BYTES, minLon, 0) < 0) && StringHelper.compare(Integer.BYTES, packedValue, Integer.BYTES, minLon2, 0) < 0) {
                        // longitude out of bounding box range
                        return;
                    }
                    int docLatitude = NumericUtils.sortableBytesToInt(packedValue, 0);
                    int docLongitude = NumericUtils.sortableBytesToInt(packedValue, Integer.BYTES);
                    if (distancePredicate.test(docLatitude, docLongitude)) {
                        adder.add(docID);
                    }
                }

                // algorithm: we create a bounding box (two bounding boxes if we cross the dateline).
                // 1. check our bounding box(es) first. if the subtree is entirely outside of those, bail.
                // 2. check if the subtree is disjoint. it may cross the bounding box but not intersect with circle
                // 3. see if the subtree is fully contained. if the subtree is enormous along the x axis, wrapping half way around the world, etc: then this can't work, just go to step 4.
                // 4. recurse naively (subtrees crossing over circle edge)
                @Override
                public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
                    if (StringHelper.compare(Integer.BYTES, minPackedValue, 0, maxLat, 0) > 0 || StringHelper.compare(Integer.BYTES, maxPackedValue, 0, minLat, 0) < 0) {
                        // latitude out of bounding box range
                        return Relation.CELL_OUTSIDE_QUERY;
                    }
                    if ((StringHelper.compare(Integer.BYTES, minPackedValue, Integer.BYTES, maxLon, 0) > 0 || StringHelper.compare(Integer.BYTES, maxPackedValue, Integer.BYTES, minLon, 0) < 0) && StringHelper.compare(Integer.BYTES, maxPackedValue, Integer.BYTES, minLon2, 0) < 0) {
                        // longitude out of bounding box range
                        return Relation.CELL_OUTSIDE_QUERY;
                    }
                    double latMin = decodeLatitude(minPackedValue, 0);
                    double lonMin = decodeLongitude(minPackedValue, Integer.BYTES);
                    double latMax = decodeLatitude(maxPackedValue, 0);
                    double lonMax = decodeLongitude(maxPackedValue, Integer.BYTES);
                    return GeoUtils.relate(latMin, latMax, lonMin, lonMax, latitude, longitude, sortKey, axisLat);
                }
            };
            final Weight weight = this;
            return new ScorerSupplier() {

                long cost = -1;

                @Override
                public Scorer get(boolean randomAccess) throws IOException {
                    values.intersect(visitor);
                    return new ConstantScoreScorer(weight, score(), result.build().iterator());
                }

                @Override
                public long cost() {
                    if (cost == -1) {
                        cost = values.estimatePointCount(visitor);
                    }
                    assert cost >= 0;
                    return cost;
                }
            };
        }
    };
}
Also used : IntersectVisitor(org.apache.lucene.index.PointValues.IntersectVisitor) LeafReader(org.apache.lucene.index.LeafReader) Rectangle(org.apache.lucene.geo.Rectangle) ConstantScoreWeight(org.apache.lucene.search.ConstantScoreWeight) Weight(org.apache.lucene.search.Weight) ConstantScoreWeight(org.apache.lucene.search.ConstantScoreWeight) PointValues(org.apache.lucene.index.PointValues) ConstantScoreScorer(org.apache.lucene.search.ConstantScoreScorer) ScorerSupplier(org.apache.lucene.search.ScorerSupplier) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) DocIdSetBuilder(org.apache.lucene.util.DocIdSetBuilder) FieldInfo(org.apache.lucene.index.FieldInfo)

Example 40 with FieldInfo

use of org.apache.lucene.index.FieldInfo in project lucene-solr by apache.

the class LatLonPointInPolygonQuery method createWeight.

@Override
public Weight createWeight(IndexSearcher searcher, boolean needsScores, float boost) throws IOException {
    // I don't use RandomAccessWeight here: it's no good to approximate with "match all docs"; this is an inverted structure and should be
    // used in the first pass:
    // bounding box over all polygons, this can speed up tree intersection/cheaply improve approximation for complex multi-polygons
    // these are pre-encoded with LatLonPoint's encoding
    final Rectangle box = Rectangle.fromPolygon(polygons);
    final byte[] minLat = new byte[Integer.BYTES];
    final byte[] maxLat = new byte[Integer.BYTES];
    final byte[] minLon = new byte[Integer.BYTES];
    final byte[] maxLon = new byte[Integer.BYTES];
    NumericUtils.intToSortableBytes(encodeLatitude(box.minLat), minLat, 0);
    NumericUtils.intToSortableBytes(encodeLatitude(box.maxLat), maxLat, 0);
    NumericUtils.intToSortableBytes(encodeLongitude(box.minLon), minLon, 0);
    NumericUtils.intToSortableBytes(encodeLongitude(box.maxLon), maxLon, 0);
    final Polygon2D tree = Polygon2D.create(polygons);
    final GeoEncodingUtils.PolygonPredicate polygonPredicate = GeoEncodingUtils.createPolygonPredicate(polygons, tree);
    return new ConstantScoreWeight(this, boost) {

        @Override
        public Scorer scorer(LeafReaderContext context) throws IOException {
            LeafReader reader = context.reader();
            PointValues values = reader.getPointValues(field);
            if (values == null) {
                // No docs in this segment had any points fields
                return null;
            }
            FieldInfo fieldInfo = reader.getFieldInfos().fieldInfo(field);
            if (fieldInfo == null) {
                // No docs in this segment indexed this field at all
                return null;
            }
            LatLonPoint.checkCompatible(fieldInfo);
            // matching docids
            DocIdSetBuilder result = new DocIdSetBuilder(reader.maxDoc(), values, field);
            values.intersect(new IntersectVisitor() {

                DocIdSetBuilder.BulkAdder adder;

                @Override
                public void grow(int count) {
                    adder = result.grow(count);
                }

                @Override
                public void visit(int docID) {
                    adder.add(docID);
                }

                @Override
                public void visit(int docID, byte[] packedValue) {
                    if (polygonPredicate.test(NumericUtils.sortableBytesToInt(packedValue, 0), NumericUtils.sortableBytesToInt(packedValue, Integer.BYTES))) {
                        adder.add(docID);
                    }
                }

                @Override
                public Relation compare(byte[] minPackedValue, byte[] maxPackedValue) {
                    if (StringHelper.compare(Integer.BYTES, minPackedValue, 0, maxLat, 0) > 0 || StringHelper.compare(Integer.BYTES, maxPackedValue, 0, minLat, 0) < 0 || StringHelper.compare(Integer.BYTES, minPackedValue, Integer.BYTES, maxLon, 0) > 0 || StringHelper.compare(Integer.BYTES, maxPackedValue, Integer.BYTES, minLon, 0) < 0) {
                        // outside of global bounding box range
                        return Relation.CELL_OUTSIDE_QUERY;
                    }
                    double cellMinLat = decodeLatitude(minPackedValue, 0);
                    double cellMinLon = decodeLongitude(minPackedValue, Integer.BYTES);
                    double cellMaxLat = decodeLatitude(maxPackedValue, 0);
                    double cellMaxLon = decodeLongitude(maxPackedValue, Integer.BYTES);
                    return tree.relate(cellMinLat, cellMaxLat, cellMinLon, cellMaxLon);
                }
            });
            return new ConstantScoreScorer(this, score(), result.build().iterator());
        }
    };
}
Also used : IntersectVisitor(org.apache.lucene.index.PointValues.IntersectVisitor) LeafReader(org.apache.lucene.index.LeafReader) GeoEncodingUtils(org.apache.lucene.geo.GeoEncodingUtils) Rectangle(org.apache.lucene.geo.Rectangle) Polygon2D(org.apache.lucene.geo.Polygon2D) ConstantScoreWeight(org.apache.lucene.search.ConstantScoreWeight) PointValues(org.apache.lucene.index.PointValues) Relation(org.apache.lucene.index.PointValues.Relation) ConstantScoreScorer(org.apache.lucene.search.ConstantScoreScorer) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) DocIdSetBuilder(org.apache.lucene.util.DocIdSetBuilder) FieldInfo(org.apache.lucene.index.FieldInfo)

Aggregations

FieldInfo (org.apache.lucene.index.FieldInfo)53 BytesRef (org.apache.lucene.util.BytesRef)13 LeafReader (org.apache.lucene.index.LeafReader)12 ArrayList (java.util.ArrayList)10 Terms (org.apache.lucene.index.Terms)9 TermsEnum (org.apache.lucene.index.TermsEnum)9 IOException (java.io.IOException)8 FieldInfos (org.apache.lucene.index.FieldInfos)8 HashMap (java.util.HashMap)7 LeafReaderContext (org.apache.lucene.index.LeafReaderContext)7 DocValuesType (org.apache.lucene.index.DocValuesType)6 PointValues (org.apache.lucene.index.PointValues)6 IndexOutput (org.apache.lucene.store.IndexOutput)6 CorruptIndexException (org.apache.lucene.index.CorruptIndexException)5 SortedSetDocValues (org.apache.lucene.index.SortedSetDocValues)5 StoredFieldVisitor (org.apache.lucene.index.StoredFieldVisitor)5 Map (java.util.Map)4 Document (org.apache.lucene.document.Document)4 EmptyDocValuesProducer (org.apache.lucene.index.EmptyDocValuesProducer)4 IndexReader (org.apache.lucene.index.IndexReader)4