Search in sources :

Example 6 with IndexNumericFieldData

use of org.opensearch.index.fielddata.IndexNumericFieldData in project OpenSearch by opensearch-project.

the class TermsSetQueryBuilder method createValuesSource.

private LongValuesSource createValuesSource(QueryShardContext context) {
    LongValuesSource longValuesSource;
    if (minimumShouldMatchField != null) {
        MappedFieldType msmFieldType = context.fieldMapper(minimumShouldMatchField);
        if (msmFieldType == null) {
            throw new QueryShardException(context, "failed to find minimum_should_match field [" + minimumShouldMatchField + "]");
        }
        IndexNumericFieldData fieldData = context.getForField(msmFieldType);
        longValuesSource = new FieldValuesSource(fieldData);
    } else if (minimumShouldMatchScript != null) {
        TermsSetQueryScript.Factory factory = context.compile(minimumShouldMatchScript, TermsSetQueryScript.CONTEXT);
        Map<String, Object> params = new HashMap<>();
        params.putAll(minimumShouldMatchScript.getParams());
        params.put("num_terms", values.size());
        longValuesSource = new ScriptLongValueSource(minimumShouldMatchScript, factory.newFactory(params, context.lookup()));
    } else {
        throw new IllegalStateException("No minimum should match has been specified");
    }
    return longValuesSource;
}
Also used : MappedFieldType(org.opensearch.index.mapper.MappedFieldType) IndexNumericFieldData(org.opensearch.index.fielddata.IndexNumericFieldData) LongValuesSource(org.apache.lucene.search.LongValuesSource) HashMap(java.util.HashMap) Map(java.util.Map)

Example 7 with IndexNumericFieldData

use of org.opensearch.index.fielddata.IndexNumericFieldData in project OpenSearch by opensearch-project.

the class ScaledFloatFieldTypeTests method testFieldData.

public void testFieldData() throws IOException {
    double scalingFactor = 0.1 + randomDouble() * 100;
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, new IndexWriterConfig(null));
    Document doc = new Document();
    doc.add(new SortedNumericDocValuesField("scaled_float1", 10));
    doc.add(new SortedNumericDocValuesField("scaled_float2", 5));
    doc.add(new SortedNumericDocValuesField("scaled_float2", 12));
    w.addDocument(doc);
    try (DirectoryReader reader = DirectoryReader.open(w)) {
        // single-valued
        ScaledFloatFieldMapper.ScaledFloatFieldType f1 = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float1", scalingFactor);
        IndexNumericFieldData fielddata = (IndexNumericFieldData) f1.fielddataBuilder("index", () -> {
            throw new UnsupportedOperationException();
        }).build(null, null);
        assertEquals(fielddata.getNumericType(), IndexNumericFieldData.NumericType.DOUBLE);
        LeafNumericFieldData leafFieldData = fielddata.load(reader.leaves().get(0));
        SortedNumericDoubleValues values = leafFieldData.getDoubleValues();
        assertTrue(values.advanceExact(0));
        assertEquals(1, values.docValueCount());
        assertEquals(10 / f1.getScalingFactor(), values.nextValue(), 10e-5);
        // multi-valued
        ScaledFloatFieldMapper.ScaledFloatFieldType f2 = new ScaledFloatFieldMapper.ScaledFloatFieldType("scaled_float2", scalingFactor);
        fielddata = (IndexNumericFieldData) f2.fielddataBuilder("index", () -> {
            throw new UnsupportedOperationException();
        }).build(null, null);
        leafFieldData = fielddata.load(reader.leaves().get(0));
        values = leafFieldData.getDoubleValues();
        assertTrue(values.advanceExact(0));
        assertEquals(2, values.docValueCount());
        assertEquals(5 / f2.getScalingFactor(), values.nextValue(), 10e-5);
        assertEquals(12 / f2.getScalingFactor(), values.nextValue(), 10e-5);
    }
    IOUtils.close(w, dir);
}
Also used : LeafNumericFieldData(org.opensearch.index.fielddata.LeafNumericFieldData) DirectoryReader(org.apache.lucene.index.DirectoryReader) IndexNumericFieldData(org.opensearch.index.fielddata.IndexNumericFieldData) Document(org.apache.lucene.document.Document) SortedNumericDocValuesField(org.apache.lucene.document.SortedNumericDocValuesField) IndexWriter(org.apache.lucene.index.IndexWriter) SortedNumericDoubleValues(org.opensearch.index.fielddata.SortedNumericDoubleValues) Directory(org.apache.lucene.store.Directory) IndexWriterConfig(org.apache.lucene.index.IndexWriterConfig)

Example 8 with IndexNumericFieldData

use of org.opensearch.index.fielddata.IndexNumericFieldData in project OpenSearch by opensearch-project.

the class SliceBuilderTests method createShardContext.

private QueryShardContext createShardContext(Version indexVersionCreated, IndexReader reader, String fieldName, DocValuesType dvType, int numShards, int shardId) {
    MappedFieldType fieldType = new MappedFieldType(fieldName, true, false, dvType != null, TextSearchInfo.NONE, Collections.emptyMap()) {

        @Override
        public ValueFetcher valueFetcher(QueryShardContext context, SearchLookup searchLookup, String format) {
            throw new UnsupportedOperationException();
        }

        @Override
        public String typeName() {
            return null;
        }

        @Override
        public Query termQuery(Object value, @Nullable QueryShardContext context) {
            return null;
        }

        public Query existsQuery(QueryShardContext context) {
            return null;
        }
    };
    QueryShardContext context = mock(QueryShardContext.class);
    when(context.fieldMapper(fieldName)).thenReturn(fieldType);
    when(context.getIndexReader()).thenReturn(reader);
    when(context.getShardId()).thenReturn(shardId);
    IndexSettings indexSettings = createIndexSettings(indexVersionCreated, numShards);
    when(context.getIndexSettings()).thenReturn(indexSettings);
    if (dvType != null) {
        IndexNumericFieldData fd = mock(IndexNumericFieldData.class);
        when(context.getForField(fieldType)).thenReturn(fd);
    }
    return context;
}
Also used : IndexSettings(org.opensearch.index.IndexSettings) MappedFieldType(org.opensearch.index.mapper.MappedFieldType) IndexNumericFieldData(org.opensearch.index.fielddata.IndexNumericFieldData) QueryShardContext(org.opensearch.index.query.QueryShardContext) Matchers.containsString(org.hamcrest.Matchers.containsString) Nullable(org.opensearch.common.Nullable) SearchLookup(org.opensearch.search.lookup.SearchLookup)

Example 9 with IndexNumericFieldData

use of org.opensearch.index.fielddata.IndexNumericFieldData in project OpenSearch by opensearch-project.

the class NumberFieldTypeTests method doTestIndexSortRangeQueries.

public void doTestIndexSortRangeQueries(NumberType type, Supplier<Number> valueSupplier) throws IOException {
    // Create index settings with an index sort.
    Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 1).put("index.sort.field", "field").build();
    IndexMetadata indexMetadata = new IndexMetadata.Builder("index").settings(settings).build();
    IndexSettings indexSettings = new IndexSettings(indexMetadata, settings);
    // Create an index writer configured with the same index sort.
    NumberFieldType fieldType = new NumberFieldType("field", type);
    IndexNumericFieldData fielddata = (IndexNumericFieldData) fieldType.fielddataBuilder("index", () -> {
        throw new UnsupportedOperationException();
    }).build(null, null);
    SortField sortField = fielddata.sortField(null, MultiValueMode.MIN, null, randomBoolean());
    IndexWriterConfig writerConfig = new IndexWriterConfig();
    writerConfig.setIndexSort(new Sort(sortField));
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, writerConfig);
    final int numDocs = TestUtil.nextInt(random(), 100, 500);
    for (int i = 0; i < numDocs; ++i) {
        w.addDocument(type.createFields("field", valueSupplier.get(), true, true, false));
    }
    // Ensure that the optimized index sort query gives the same results as a points query.
    DirectoryReader reader = DirectoryReader.open(w);
    IndexSearcher searcher = newSearcher(reader);
    QueryShardContext context = new QueryShardContext(0, indexSettings, BigArrays.NON_RECYCLING_INSTANCE, null, null, null, null, null, xContentRegistry(), writableRegistry(), null, null, () -> 0L, null, null, () -> true, null);
    final int iters = 10;
    for (int iter = 0; iter < iters; ++iter) {
        Query query = type.rangeQuery("field", random().nextBoolean() ? null : valueSupplier.get(), random().nextBoolean() ? null : valueSupplier.get(), randomBoolean(), randomBoolean(), true, context);
        assertThat(query, instanceOf(IndexSortSortedNumericDocValuesRangeQuery.class));
        Query fallbackQuery = ((IndexSortSortedNumericDocValuesRangeQuery) query).getFallbackQuery();
        assertThat(fallbackQuery, instanceOf(IndexOrDocValuesQuery.class));
        IndexOrDocValuesQuery indexOrDvQuery = (IndexOrDocValuesQuery) fallbackQuery;
        assertEquals(searcher.count(query), searcher.count(indexOrDvQuery.getIndexQuery()));
    }
    reader.close();
    w.close();
    dir.close();
}
Also used : IndexSearcher(org.apache.lucene.search.IndexSearcher) Query(org.apache.lucene.search.Query) MatchNoDocsQuery(org.apache.lucene.search.MatchNoDocsQuery) IndexOrDocValuesQuery(org.apache.lucene.search.IndexOrDocValuesQuery) IndexSortSortedNumericDocValuesRangeQuery(org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery) DirectoryReader(org.apache.lucene.index.DirectoryReader) NumberFieldType(org.opensearch.index.mapper.NumberFieldMapper.NumberFieldType) XContentBuilder(org.opensearch.common.xcontent.XContentBuilder) IndexSettings(org.opensearch.index.IndexSettings) IndexNumericFieldData(org.opensearch.index.fielddata.IndexNumericFieldData) SortField(org.apache.lucene.search.SortField) LongPoint(org.apache.lucene.document.LongPoint) DoublePoint(org.apache.lucene.document.DoublePoint) HalfFloatPoint(org.apache.lucene.sandbox.document.HalfFloatPoint) IntPoint(org.apache.lucene.document.IntPoint) FloatPoint(org.apache.lucene.document.FloatPoint) IndexWriter(org.apache.lucene.index.IndexWriter) Sort(org.apache.lucene.search.Sort) QueryShardContext(org.opensearch.index.query.QueryShardContext) IndexOrDocValuesQuery(org.apache.lucene.search.IndexOrDocValuesQuery) IndexMetadata(org.opensearch.cluster.metadata.IndexMetadata) Settings(org.opensearch.common.settings.Settings) IndexSettings(org.opensearch.index.IndexSettings) IndexSortSortedNumericDocValuesRangeQuery(org.apache.lucene.sandbox.search.IndexSortSortedNumericDocValuesRangeQuery) IndexWriterConfig(org.apache.lucene.index.IndexWriterConfig) Directory(org.apache.lucene.store.Directory)

Example 10 with IndexNumericFieldData

use of org.opensearch.index.fielddata.IndexNumericFieldData in project OpenSearch by opensearch-project.

the class FieldSortBuilder method build.

@Override
public SortFieldAndFormat build(QueryShardContext context) throws IOException {
    if (DOC_FIELD_NAME.equals(fieldName)) {
        return order == SortOrder.DESC ? SORT_DOC_REVERSE : SORT_DOC;
    }
    MappedFieldType fieldType = context.fieldMapper(fieldName);
    Nested nested = nested(context, fieldType);
    if (fieldType == null) {
        fieldType = resolveUnmappedType(context);
    }
    boolean reverse = order == SortOrder.DESC;
    IndexFieldData<?> fieldData = context.getForField(fieldType);
    if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
        throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields");
    }
    final SortField field;
    boolean isNanosecond = false;
    if (numericType != null) {
        if (fieldData instanceof IndexNumericFieldData == false) {
            throw new QueryShardException(context, "[numeric_type] option cannot be set on a non-numeric field, got " + fieldType.typeName());
        }
        IndexNumericFieldData numericFieldData = (IndexNumericFieldData) fieldData;
        NumericType resolvedType = resolveNumericType(numericType);
        field = numericFieldData.sortField(resolvedType, missing, localSortMode(), nested, reverse);
        isNanosecond = resolvedType == NumericType.DATE_NANOSECONDS;
    } else {
        field = fieldData.sortField(missing, localSortMode(), nested, reverse);
        if (fieldData instanceof IndexNumericFieldData) {
            isNanosecond = ((IndexNumericFieldData) fieldData).getNumericType() == NumericType.DATE_NANOSECONDS;
        }
    }
    DocValueFormat format = fieldType.docValueFormat(null, null);
    if (isNanosecond) {
        format = DocValueFormat.withNanosecondResolution(format);
    }
    return new SortFieldAndFormat(field, format);
}
Also used : NumericType(org.opensearch.index.fielddata.IndexNumericFieldData.NumericType) DocValueFormat(org.opensearch.search.DocValueFormat) MappedFieldType(org.opensearch.index.mapper.MappedFieldType) Nested(org.opensearch.index.fielddata.IndexFieldData.XFieldComparatorSource.Nested) IndexNumericFieldData(org.opensearch.index.fielddata.IndexNumericFieldData) QueryShardException(org.opensearch.index.query.QueryShardException) SortField(org.apache.lucene.search.SortField)

Aggregations

IndexNumericFieldData (org.opensearch.index.fielddata.IndexNumericFieldData)14 MappedFieldType (org.opensearch.index.mapper.MappedFieldType)7 ParseException (java.text.ParseException)4 LeafNumericFieldData (org.opensearch.index.fielddata.LeafNumericFieldData)4 SortedNumericDoubleValues (org.opensearch.index.fielddata.SortedNumericDoubleValues)4 SearchLookup (org.opensearch.search.lookup.SearchLookup)4 IOException (java.io.IOException)3 Collections (java.util.Collections)3 Mockito.any (org.mockito.Mockito.any)3 Mockito.anyInt (org.mockito.Mockito.anyInt)3 Mockito.mock (org.mockito.Mockito.mock)3 Mockito.when (org.mockito.Mockito.when)3 MapperService (org.opensearch.index.mapper.MapperService)3 NumberFieldType (org.opensearch.index.mapper.NumberFieldMapper.NumberFieldType)3 ScriptException (org.opensearch.script.ScriptException)3 OpenSearchTestCase (org.opensearch.test.OpenSearchTestCase)3 DirectoryReader (org.apache.lucene.index.DirectoryReader)2 IndexWriter (org.apache.lucene.index.IndexWriter)2 IndexWriterConfig (org.apache.lucene.index.IndexWriterConfig)2 MatchNoDocsQuery (org.apache.lucene.search.MatchNoDocsQuery)2