Examples with Nested org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested used on opensource projects

Example 1 with Nested

use of org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested in project elasticsearch by elastic.

the class AbstractStringFieldDataTestCase method testNestedSorting.

public void testNestedSorting(MultiValueMode sortMode) throws IOException {
    final String[] values = new String[randomIntBetween(2, 20)];
    for (int i = 0; i < values.length; ++i) {
        values[i] = TestUtil.randomSimpleString(random());
    }
    final int numParents = scaledRandomIntBetween(10, 3072);
    List<Document> docs = new ArrayList<>();
    FixedBitSet parents = new FixedBitSet(64);
    for (int i = 0; i < numParents; ++i) {
        docs.clear();
        final int numChildren = randomInt(4);
        for (int j = 0; j < numChildren; ++j) {
            final Document child = new Document();
            final int numValues = randomInt(3);
            for (int k = 0; k < numValues; ++k) {
                final String value = RandomPicks.randomFrom(random(), values);
                addField(child, "text", value);
            }
            docs.add(child);
        }
        final Document parent = new Document();
        parent.add(new StringField("type", "parent", Store.YES));
        final String value = RandomPicks.randomFrom(random(), values);
        if (value != null) {
            addField(parent, "text", value);
        }
        docs.add(parent);
        int bit = parents.prevSetBit(parents.length() - 1) + docs.size();
        parents = FixedBitSet.ensureCapacity(parents, bit);
        parents.set(bit);
        writer.addDocuments(docs);
        if (randomInt(10) == 0) {
            writer.commit();
        }
    }
    DirectoryReader directoryReader = DirectoryReader.open(writer);
    directoryReader = ElasticsearchDirectoryReader.wrap(directoryReader, new ShardId(indexService.index(), 0));
    IndexSearcher searcher = new IndexSearcher(directoryReader);
    IndexFieldData<?> fieldData = getForField("text");
    final Object missingValue;
    switch(randomInt(4)) {
        case 0:
            missingValue = "_first";
            break;
        case 1:
            missingValue = "_last";
            break;
        case 2:
            missingValue = new BytesRef(RandomPicks.randomFrom(random(), values));
            break;
        default:
            missingValue = new BytesRef(TestUtil.randomSimpleString(random()));
            break;
    }
    Query parentFilter = new TermQuery(new Term("type", "parent"));
    Query childFilter = Queries.not(parentFilter);
    Nested nested = createNested(searcher, parentFilter, childFilter);
    BytesRefFieldComparatorSource nestedComparatorSource = new BytesRefFieldComparatorSource(fieldData, missingValue, sortMode, nested);
    ToParentBlockJoinQuery query = new ToParentBlockJoinQuery(new ConstantScoreQuery(childFilter), new QueryBitSetProducer(parentFilter), ScoreMode.None);
    Sort sort = new Sort(new SortField("text", nestedComparatorSource));
    TopFieldDocs topDocs = searcher.search(query, randomIntBetween(1, numParents), sort);
    assertTrue(topDocs.scoreDocs.length > 0);
    BytesRef previous = null;
    for (int i = 0; i < topDocs.scoreDocs.length; ++i) {
        final int docID = topDocs.scoreDocs[i].doc;
        assertTrue("expected " + docID + " to be a parent", parents.get(docID));
        BytesRef cmpValue = null;
        for (int child = parents.prevSetBit(docID - 1) + 1; child < docID; ++child) {
            String[] sVals = searcher.doc(child).getValues("text");
            final BytesRef[] vals;
            if (sVals.length == 0) {
                vals = new BytesRef[0];
            } else {
                vals = new BytesRef[sVals.length];
                for (int j = 0; j < vals.length; ++j) {
                    vals[j] = new BytesRef(sVals[j]);
                }
            }
            for (BytesRef value : vals) {
                if (cmpValue == null) {
                    cmpValue = value;
                } else if (sortMode == MultiValueMode.MIN && value.compareTo(cmpValue) < 0) {
                    cmpValue = value;
                } else if (sortMode == MultiValueMode.MAX && value.compareTo(cmpValue) > 0) {
                    cmpValue = value;
                }
            }
        }
        if (cmpValue == null) {
            if ("_first".equals(missingValue)) {
                cmpValue = new BytesRef();
            } else if ("_last".equals(missingValue) == false) {
                cmpValue = (BytesRef) missingValue;
            }
        }
        if (previous != null && cmpValue != null) {
            assertTrue(previous.utf8ToString() + "   /   " + cmpValue.utf8ToString(), previous.compareTo(cmpValue) <= 0);
        }
        previous = cmpValue;
    }
    searcher.getIndexReader().close();
}

Example 2 with Nested

use of org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested in project elasticsearch by elastic.

the class FieldSortBuilder method build.

@Override
public SortFieldAndFormat build(QueryShardContext context) throws IOException {
    if (DOC_FIELD_NAME.equals(fieldName)) {
        if (order == SortOrder.DESC) {
            return SORT_DOC_REVERSE;
        } else {
            return SORT_DOC;
        }
    } else {
        MappedFieldType fieldType = context.fieldMapper(fieldName);
        if (fieldType == null) {
            if (unmappedType != null) {
                fieldType = context.getMapperService().unmappedFieldType(unmappedType);
            } else {
                throw new QueryShardException(context, "No mapping found for [" + fieldName + "] in order to sort on");
            }
        }
        MultiValueMode localSortMode = null;
        if (sortMode != null) {
            localSortMode = MultiValueMode.fromString(sortMode.toString());
        }
        boolean reverse = (order == SortOrder.DESC);
        if (localSortMode == null) {
            localSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
        }
        final Nested nested = resolveNested(context, nestedPath, nestedFilter);
        IndexFieldData<?> fieldData = context.getForField(fieldType);
        if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
            throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields");
        }
        IndexFieldData.XFieldComparatorSource fieldComparatorSource = fieldData.comparatorSource(missing, localSortMode, nested);
        SortField field = new SortField(fieldType.name(), fieldComparatorSource, reverse);
        return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null));
    }
}

Example 3 with Nested

use of org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested in project elasticsearch by elastic.

the class GeoDistanceSortBuilder method build.

@Override
public SortFieldAndFormat build(QueryShardContext context) throws IOException {
    final boolean indexCreatedBeforeV2_0 = context.indexVersionCreated().before(Version.V_2_0_0);
    // validation was not available prior to 2.x, so to support bwc percolation queries we only ignore_malformed
    // on 2.x created indexes
    GeoPoint[] localPoints = points.toArray(new GeoPoint[points.size()]);
    if (!indexCreatedBeforeV2_0 && !GeoValidationMethod.isIgnoreMalformed(validation)) {
        for (GeoPoint point : localPoints) {
            if (GeoUtils.isValidLatitude(point.lat()) == false) {
                throw new ElasticsearchParseException("illegal latitude value [{}] for [GeoDistanceSort] for field [{}].", point.lat(), fieldName);
            }
            if (GeoUtils.isValidLongitude(point.lon()) == false) {
                throw new ElasticsearchParseException("illegal longitude value [{}] for [GeoDistanceSort] for field [{}].", point.lon(), fieldName);
            }
        }
    }
    if (GeoValidationMethod.isCoerce(validation)) {
        for (GeoPoint point : localPoints) {
            GeoUtils.normalizePoint(point, true, true);
        }
    }
    boolean reverse = (order == SortOrder.DESC);
    final MultiValueMode finalSortMode;
    if (sortMode == null) {
        finalSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
    } else {
        finalSortMode = MultiValueMode.fromString(sortMode.toString());
    }
    MappedFieldType fieldType = context.fieldMapper(fieldName);
    if (fieldType == null) {
        throw new IllegalArgumentException("failed to find mapper for [" + fieldName + "] for geo distance based sort");
    }
    final IndexGeoPointFieldData geoIndexFieldData = context.getForField(fieldType);
    final Nested nested = resolveNested(context, nestedPath, nestedFilter);
    if (// only works with 5.x geo_point
    geoIndexFieldData.getClass() == LatLonPointDVIndexFieldData.class && nested == null && // LatLonDocValuesField internally picks the closest point
    finalSortMode == MultiValueMode.MIN && unit == DistanceUnit.METERS && reverse == false && localPoints.length == 1) {
        return new SortFieldAndFormat(LatLonDocValuesField.newDistanceSort(fieldName, localPoints[0].lat(), localPoints[0].lon()), DocValueFormat.RAW);
    }
    IndexFieldData.XFieldComparatorSource geoDistanceComparatorSource = new IndexFieldData.XFieldComparatorSource() {

        @Override
        public SortField.Type reducedType() {
            return SortField.Type.DOUBLE;
        }

        @Override
        public FieldComparator<?> newComparator(String fieldname, int numHits, int sortPos, boolean reversed) {
            return new FieldComparator.DoubleComparator(numHits, null, null) {

                @Override
                protected NumericDocValues getNumericDocValues(LeafReaderContext context, String field) throws IOException {
                    final MultiGeoPointValues geoPointValues = geoIndexFieldData.load(context).getGeoPointValues();
                    final SortedNumericDoubleValues distanceValues = GeoUtils.distanceValues(geoDistance, unit, geoPointValues, localPoints);
                    final NumericDoubleValues selectedValues;
                    if (nested == null) {
                        selectedValues = finalSortMode.select(distanceValues, Double.POSITIVE_INFINITY);
                    } else {
                        final BitSet rootDocs = nested.rootDocs(context);
                        final DocIdSetIterator innerDocs = nested.innerDocs(context);
                        selectedValues = finalSortMode.select(distanceValues, Double.POSITIVE_INFINITY, rootDocs, innerDocs, context.reader().maxDoc());
                    }
                    return selectedValues.getRawDoubleValues();
                }
            };
        }
    };
    return new SortFieldAndFormat(new SortField(fieldName, geoDistanceComparatorSource, reverse), DocValueFormat.RAW);
}

Example 4 with Nested

use of org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested in project elasticsearch by elastic.

the class ScriptSortBuilder method build.

@Override
public SortFieldAndFormat build(QueryShardContext context) throws IOException {
    final SearchScript searchScript = context.getSearchScript(script, ScriptContext.Standard.SEARCH);
    MultiValueMode valueMode = null;
    if (sortMode != null) {
        valueMode = MultiValueMode.fromString(sortMode.toString());
    }
    boolean reverse = (order == SortOrder.DESC);
    if (valueMode == null) {
        valueMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
    }
    final Nested nested = resolveNested(context, nestedPath, nestedFilter);
    final IndexFieldData.XFieldComparatorSource fieldComparatorSource;
    switch(type) {
        case STRING:
            fieldComparatorSource = new BytesRefFieldComparatorSource(null, null, valueMode, nested) {

                LeafSearchScript leafScript;

                @Override
                protected SortedBinaryDocValues getValues(LeafReaderContext context) throws IOException {
                    leafScript = searchScript.getLeafSearchScript(context);
                    final BinaryDocValues values = new BinaryDocValues() {

                        final BytesRefBuilder spare = new BytesRefBuilder();

                        @Override
                        public BytesRef get(int docID) {
                            leafScript.setDocument(docID);
                            spare.copyChars(leafScript.run().toString());
                            return spare.get();
                        }
                    };
                    return FieldData.singleton(values, null);
                }

                @Override
                protected void setScorer(Scorer scorer) {
                    leafScript.setScorer(scorer);
                }
            };
            break;
        case NUMBER:
            fieldComparatorSource = new DoubleValuesComparatorSource(null, Double.MAX_VALUE, valueMode, nested) {

                LeafSearchScript leafScript;

                @Override
                protected SortedNumericDoubleValues getValues(LeafReaderContext context) throws IOException {
                    leafScript = searchScript.getLeafSearchScript(context);
                    final NumericDoubleValues values = new NumericDoubleValues() {

                        @Override
                        public double get(int docID) {
                            leafScript.setDocument(docID);
                            return leafScript.runAsDouble();
                        }
                    };
                    return FieldData.singleton(values, null);
                }

                @Override
                protected void setScorer(Scorer scorer) {
                    leafScript.setScorer(scorer);
                }
            };
            break;
        default:
            throw new QueryShardException(context, "custom script sort type [" + type + "] not supported");
    }
    return new SortFieldAndFormat(new SortField("_script", fieldComparatorSource, reverse), DocValueFormat.RAW);
}

Example 5 with Nested

use of org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested in project elasticsearch by elastic.

the class SortBuilder method resolveNested.

protected static Nested resolveNested(QueryShardContext context, String nestedPath, QueryBuilder nestedFilter) throws IOException {
    Nested nested = null;
    if (nestedPath != null) {
        BitSetProducer rootDocumentsFilter = context.bitsetFilter(Queries.newNonNestedFilter());
        ObjectMapper nestedObjectMapper = context.getObjectMapper(nestedPath);
        if (nestedObjectMapper == null) {
            throw new QueryShardException(context, "[nested] failed to find nested object under path [" + nestedPath + "]");
        }
        if (!nestedObjectMapper.nested().isNested()) {
            throw new QueryShardException(context, "[nested] nested object under path [" + nestedPath + "] is not of nested type");
        }
        Query innerDocumentsQuery;
        if (nestedFilter != null) {
            context.nestedScope().nextLevel(nestedObjectMapper);
            innerDocumentsQuery = QueryBuilder.rewriteQuery(nestedFilter, context).toFilter(context);
            context.nestedScope().previousLevel();
        } else {
            innerDocumentsQuery = nestedObjectMapper.nestedTypeFilter();
        }
        nested = new Nested(rootDocumentsFilter, innerDocumentsQuery);
    }
    return nested;
}