Examples with IndexNumericFieldData org.elasticsearch.index.fielddata.IndexNumericFieldData used on opensource projects

Example 6 with IndexNumericFieldData

use of org.elasticsearch.index.fielddata.IndexNumericFieldData in project elasticsearch by elastic.

the class DecayFunctionBuilder method parseDateVariable.

private AbstractDistanceScoreFunction parseDateVariable(XContentParser parser, QueryShardContext context, MappedFieldType dateFieldType, MultiValueMode mode) throws IOException {
    XContentParser.Token token;
    String parameterName = null;
    String scaleString = null;
    String originString = null;
    String offsetString = "0d";
    double decay = 0.5;
    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
        if (token == XContentParser.Token.FIELD_NAME) {
            parameterName = parser.currentName();
        } else if (DecayFunctionBuilder.SCALE.equals(parameterName)) {
            scaleString = parser.text();
        } else if (DecayFunctionBuilder.ORIGIN.equals(parameterName)) {
            originString = parser.text();
        } else if (DecayFunctionBuilder.DECAY.equals(parameterName)) {
            decay = parser.doubleValue();
        } else if (DecayFunctionBuilder.OFFSET.equals(parameterName)) {
            offsetString = parser.text();
        } else {
            throw new ElasticsearchParseException("parameter [{}] not supported!", parameterName);
        }
    }
    long origin;
    if (originString == null) {
        origin = context.nowInMillis();
    } else {
        origin = ((DateFieldMapper.DateFieldType) dateFieldType).parseToMilliseconds(originString, false, null, null, context);
    }
    if (scaleString == null) {
        throw new ElasticsearchParseException("[{}] must be set for date fields.", DecayFunctionBuilder.SCALE);
    }
    TimeValue val = TimeValue.parseTimeValue(scaleString, TimeValue.timeValueHours(24), DecayFunctionParser.class.getSimpleName() + ".scale");
    double scale = val.getMillis();
    val = TimeValue.parseTimeValue(offsetString, TimeValue.timeValueHours(24), DecayFunctionParser.class.getSimpleName() + ".offset");
    double offset = val.getMillis();
    IndexNumericFieldData numericFieldData = context.getForField(dateFieldType);
    return new NumericFieldDataScoreFunction(origin, scale, decay, offset, getDecayFunction(), numericFieldData, mode);
}

Example 7 with IndexNumericFieldData

use of org.elasticsearch.index.fielddata.IndexNumericFieldData in project elasticsearch by elastic.

the class DecayFunctionBuilder method parseNumberVariable.

private AbstractDistanceScoreFunction parseNumberVariable(XContentParser parser, QueryShardContext context, MappedFieldType fieldType, MultiValueMode mode) throws IOException {
    XContentParser.Token token;
    String parameterName = null;
    double scale = 0;
    double origin = 0;
    double decay = 0.5;
    double offset = 0.0d;
    boolean scaleFound = false;
    boolean refFound = false;
    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
        if (token == XContentParser.Token.FIELD_NAME) {
            parameterName = parser.currentName();
        } else if (DecayFunctionBuilder.SCALE.equals(parameterName)) {
            scale = parser.doubleValue();
            scaleFound = true;
        } else if (DecayFunctionBuilder.DECAY.equals(parameterName)) {
            decay = parser.doubleValue();
        } else if (DecayFunctionBuilder.ORIGIN.equals(parameterName)) {
            origin = parser.doubleValue();
            refFound = true;
        } else if (DecayFunctionBuilder.OFFSET.equals(parameterName)) {
            offset = parser.doubleValue();
        } else {
            throw new ElasticsearchParseException("parameter [{}] not supported!", parameterName);
        }
    }
    if (!scaleFound || !refFound) {
        throw new ElasticsearchParseException("both [{}] and [{}] must be set for numeric fields.", DecayFunctionBuilder.SCALE, DecayFunctionBuilder.ORIGIN);
    }
    IndexNumericFieldData numericFieldData = context.getForField(fieldType);
    return new NumericFieldDataScoreFunction(origin, scale, decay, offset, getDecayFunction(), numericFieldData, mode);
}

Example 8 with IndexNumericFieldData

use of org.elasticsearch.index.fielddata.IndexNumericFieldData in project elasticsearch by elastic.

the class SliceBuilderTests method testToFilter.

public void testToFilter() throws IOException {
    Directory dir = new RAMDirectory();
    try (IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig(new MockAnalyzer(random())))) {
        writer.commit();
    }
    QueryShardContext context = mock(QueryShardContext.class);
    try (IndexReader reader = DirectoryReader.open(dir)) {
        MappedFieldType fieldType = new MappedFieldType() {

            @Override
            public MappedFieldType clone() {
                return null;
            }

            @Override
            public String typeName() {
                return null;
            }

            @Override
            public Query termQuery(Object value, @Nullable QueryShardContext context) {
                return null;
            }
        };
        fieldType.setName(UidFieldMapper.NAME);
        fieldType.setHasDocValues(false);
        when(context.fieldMapper(UidFieldMapper.NAME)).thenReturn(fieldType);
        when(context.getIndexReader()).thenReturn(reader);
        SliceBuilder builder = new SliceBuilder(5, 10);
        Query query = builder.toFilter(context, 0, 1);
        assertThat(query, instanceOf(TermsSliceQuery.class));
        assertThat(builder.toFilter(context, 0, 1), equalTo(query));
        try (IndexReader newReader = DirectoryReader.open(dir)) {
            when(context.getIndexReader()).thenReturn(newReader);
            assertThat(builder.toFilter(context, 0, 1), equalTo(query));
        }
    }
    try (IndexReader reader = DirectoryReader.open(dir)) {
        MappedFieldType fieldType = new MappedFieldType() {

            @Override
            public MappedFieldType clone() {
                return null;
            }

            @Override
            public String typeName() {
                return null;
            }

            @Override
            public Query termQuery(Object value, @Nullable QueryShardContext context) {
                return null;
            }
        };
        fieldType.setName("field_doc_values");
        fieldType.setHasDocValues(true);
        fieldType.setDocValuesType(DocValuesType.SORTED_NUMERIC);
        when(context.fieldMapper("field_doc_values")).thenReturn(fieldType);
        when(context.getIndexReader()).thenReturn(reader);
        IndexNumericFieldData fd = mock(IndexNumericFieldData.class);
        when(context.getForField(fieldType)).thenReturn(fd);
        SliceBuilder builder = new SliceBuilder("field_doc_values", 5, 10);
        Query query = builder.toFilter(context, 0, 1);
        assertThat(query, instanceOf(DocValuesSliceQuery.class));
        assertThat(builder.toFilter(context, 0, 1), equalTo(query));
        try (IndexReader newReader = DirectoryReader.open(dir)) {
            when(context.getIndexReader()).thenReturn(newReader);
            assertThat(builder.toFilter(context, 0, 1), equalTo(query));
        }
        // numSlices > numShards
        int numSlices = randomIntBetween(10, 100);
        int numShards = randomIntBetween(1, 9);
        Map<Integer, AtomicInteger> numSliceMap = new HashMap<>();
        for (int i = 0; i < numSlices; i++) {
            for (int j = 0; j < numShards; j++) {
                SliceBuilder slice = new SliceBuilder("_uid", i, numSlices);
                Query q = slice.toFilter(context, j, numShards);
                if (q instanceof TermsSliceQuery || q instanceof MatchAllDocsQuery) {
                    AtomicInteger count = numSliceMap.get(j);
                    if (count == null) {
                        count = new AtomicInteger(0);
                        numSliceMap.put(j, count);
                    }
                    count.incrementAndGet();
                    if (q instanceof MatchAllDocsQuery) {
                        assertThat(count.get(), equalTo(1));
                    }
                } else {
                    assertThat(q, instanceOf(MatchNoDocsQuery.class));
                }
            }
        }
        int total = 0;
        for (Map.Entry<Integer, AtomicInteger> e : numSliceMap.entrySet()) {
            total += e.getValue().get();
        }
        assertThat(total, equalTo(numSlices));
        // numShards > numSlices
        numShards = randomIntBetween(4, 100);
        numSlices = randomIntBetween(2, numShards - 1);
        List<Integer> targetShards = new ArrayList<>();
        for (int i = 0; i < numSlices; i++) {
            for (int j = 0; j < numShards; j++) {
                SliceBuilder slice = new SliceBuilder("_uid", i, numSlices);
                Query q = slice.toFilter(context, j, numShards);
                if (q instanceof MatchNoDocsQuery == false) {
                    assertThat(q, instanceOf(MatchAllDocsQuery.class));
                    targetShards.add(j);
                }
            }
        }
        assertThat(targetShards.size(), equalTo(numShards));
        assertThat(new HashSet<>(targetShards).size(), equalTo(numShards));
        // numShards == numSlices
        numShards = randomIntBetween(2, 10);
        numSlices = numShards;
        for (int i = 0; i < numSlices; i++) {
            for (int j = 0; j < numShards; j++) {
                SliceBuilder slice = new SliceBuilder("_uid", i, numSlices);
                Query q = slice.toFilter(context, j, numShards);
                if (i == j) {
                    assertThat(q, instanceOf(MatchAllDocsQuery.class));
                } else {
                    assertThat(q, instanceOf(MatchNoDocsQuery.class));
                }
            }
        }
    }
    try (IndexReader reader = DirectoryReader.open(dir)) {
        MappedFieldType fieldType = new MappedFieldType() {

            @Override
            public MappedFieldType clone() {
                return null;
            }

            @Override
            public String typeName() {
                return null;
            }

            @Override
            public Query termQuery(Object value, @Nullable QueryShardContext context) {
                return null;
            }
        };
        fieldType.setName("field_without_doc_values");
        when(context.fieldMapper("field_without_doc_values")).thenReturn(fieldType);
        when(context.getIndexReader()).thenReturn(reader);
        SliceBuilder builder = new SliceBuilder("field_without_doc_values", 5, 10);
        IllegalArgumentException exc = expectThrows(IllegalArgumentException.class, () -> builder.toFilter(context, 0, 1));
        assertThat(exc.getMessage(), containsString("cannot load numeric doc values"));
    }
}

Example 9 with IndexNumericFieldData

use of org.elasticsearch.index.fielddata.IndexNumericFieldData in project elasticsearch by elastic.

the class ExpressionScriptEngineService method search.

@Override
public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) {
    Expression expr = (Expression) compiledScript.compiled();
    MapperService mapper = lookup.doc().mapperService();
    // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings,
    // instead of complicating SimpleBindings (which should stay simple)
    SimpleBindings bindings = new SimpleBindings();
    ReplaceableConstDoubleValueSource specialValue = null;
    boolean needsScores = false;
    for (String variable : expr.variables) {
        try {
            if (variable.equals("_score")) {
                bindings.add(new SortField("_score", SortField.Type.SCORE));
                needsScores = true;
            } else if (variable.equals("_value")) {
                specialValue = new ReplaceableConstDoubleValueSource();
                bindings.add("_value", specialValue);
            // noop: _value is special for aggregations, and is handled in ExpressionScriptBindings
            // TODO: if some uses it in a scoring expression, they will get a nasty failure when evaluating...need a
            // way to know this is for aggregations and so _value is ok to have...
            } else if (vars != null && vars.containsKey(variable)) {
                // TODO: document and/or error if vars contains _score?
                // NOTE: by checking for the variable in vars first, it allows masking document fields with a global constant,
                // but if we were to reverse it, we could provide a way to supply dynamic defaults for documents missing the field?
                Object value = vars.get(variable);
                if (value instanceof Number) {
                    bindings.add(variable, new DoubleConstValueSource(((Number) value).doubleValue()).asDoubleValuesSource());
                } else {
                    throw new ParseException("Parameter [" + variable + "] must be a numeric type", 0);
                }
            } else {
                String fieldname = null;
                String methodname = null;
                // .value is the default for doc['field'], its optional.
                String variablename = "value";
                // true if the variable is of type doc['field'].date.xxx
                boolean dateAccessor = false;
                VariableContext[] parts = VariableContext.parse(variable);
                if (parts[0].text.equals("doc") == false) {
                    throw new ParseException("Unknown variable [" + parts[0].text + "]", 0);
                }
                if (parts.length < 2 || parts[1].type != VariableContext.Type.STR_INDEX) {
                    throw new ParseException("Variable 'doc' must be used with a specific field like: doc['myfield']", 3);
                } else {
                    fieldname = parts[1].text;
                }
                if (parts.length == 3) {
                    if (parts[2].type == VariableContext.Type.METHOD) {
                        methodname = parts[2].text;
                    } else if (parts[2].type == VariableContext.Type.MEMBER) {
                        variablename = parts[2].text;
                    } else {
                        throw new IllegalArgumentException("Only member variables or member methods may be accessed on a field when not accessing the field directly");
                    }
                }
                if (parts.length > 3) {
                    // access to the .date "object" within the field
                    if (parts.length == 4 && ("date".equals(parts[2].text) || "getDate".equals(parts[2].text))) {
                        if (parts[3].type == VariableContext.Type.METHOD) {
                            methodname = parts[3].text;
                            dateAccessor = true;
                        } else if (parts[3].type == VariableContext.Type.MEMBER) {
                            variablename = parts[3].text;
                            dateAccessor = true;
                        }
                    }
                    if (!dateAccessor) {
                        throw new IllegalArgumentException("Variable [" + variable + "] does not follow an allowed format of either doc['field'] or doc['field'].method()");
                    }
                }
                MappedFieldType fieldType = mapper.fullName(fieldname);
                if (fieldType == null) {
                    throw new ParseException("Field [" + fieldname + "] does not exist in mappings", 5);
                }
                IndexFieldData<?> fieldData = lookup.doc().fieldDataService().getForField(fieldType);
                // delegate valuesource creation based on field's type
                // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
                final ValueSource valueSource;
                if (fieldType instanceof GeoPointFieldType) {
                    // geo
                    if (methodname == null) {
                        valueSource = GeoField.getVariable(fieldData, fieldname, variablename);
                    } else {
                        valueSource = GeoField.getMethod(fieldData, fieldname, methodname);
                    }
                } else if (fieldType instanceof DateFieldMapper.DateFieldType) {
                    if (dateAccessor) {
                        // date object
                        if (methodname == null) {
                            valueSource = DateObject.getVariable(fieldData, fieldname, variablename);
                        } else {
                            valueSource = DateObject.getMethod(fieldData, fieldname, methodname);
                        }
                    } else {
                        // date field itself
                        if (methodname == null) {
                            valueSource = DateField.getVariable(fieldData, fieldname, variablename);
                        } else {
                            valueSource = DateField.getMethod(fieldData, fieldname, methodname);
                        }
                    }
                } else if (fieldData instanceof IndexNumericFieldData) {
                    // number
                    if (methodname == null) {
                        valueSource = NumericField.getVariable(fieldData, fieldname, variablename);
                    } else {
                        valueSource = NumericField.getMethod(fieldData, fieldname, methodname);
                    }
                } else {
                    throw new ParseException("Field [" + fieldname + "] must be numeric, date, or geopoint", 5);
                }
                needsScores |= valueSource.getSortField(false).needsScores();
                bindings.add(variable, valueSource.asDoubleValuesSource());
            }
        } catch (Exception e) {
            // we defer "binding" of variables until here: give context for that variable
            throw convertToScriptException("link error", expr.sourceText, variable, e);
        }
    }
    return new ExpressionSearchScript(compiledScript, bindings, specialValue, needsScores);
}

Example 10 with IndexNumericFieldData

use of org.elasticsearch.index.fielddata.IndexNumericFieldData in project elasticsearch by elastic.

the class MultiValuesSourceAggregationBuilder method config.

public ValuesSourceConfig<VS> config(SearchContext context, String field, Script script) {
    ValueType valueType = this.valueType != null ? this.valueType : targetValueType;
    if (field == null) {
        if (script == null) {
            ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(ValuesSourceType.ANY);
            return config.format(resolveFormat(null, valueType));
        }
        ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : this.valuesSourceType;
        if (valuesSourceType == null || valuesSourceType == ValuesSourceType.ANY) {
            // the specific value source type is undefined, but for scripts,
            // we need to have a specific value source
            // type to know how to handle the script values, so we fallback
            // on Bytes
            valuesSourceType = ValuesSourceType.BYTES;
        }
        ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(valuesSourceType);
        config.missing(missingMap.get(field));
        return config.format(resolveFormat(format, valueType));
    }
    MappedFieldType fieldType = context.smartNameFieldType(field);
    if (fieldType == null) {
        ValuesSourceType valuesSourceType = valueType != null ? valueType.getValuesSourceType() : this.valuesSourceType;
        ValuesSourceConfig<VS> config = new ValuesSourceConfig<>(valuesSourceType);
        config.missing(missingMap.get(field));
        config.format(resolveFormat(format, valueType));
        return config.unmapped(true);
    }
    IndexFieldData<?> indexFieldData = context.fieldData().getForField(fieldType);
    ValuesSourceConfig<VS> config;
    if (valuesSourceType == ValuesSourceType.ANY) {
        if (indexFieldData instanceof IndexNumericFieldData) {
            config = new ValuesSourceConfig<>(ValuesSourceType.NUMERIC);
        } else if (indexFieldData instanceof IndexGeoPointFieldData) {
            config = new ValuesSourceConfig<>(ValuesSourceType.GEOPOINT);
        } else {
            config = new ValuesSourceConfig<>(ValuesSourceType.BYTES);
        }
    } else {
        config = new ValuesSourceConfig<>(valuesSourceType);
    }
    config.fieldContext(new FieldContext(field, indexFieldData, fieldType));
    config.missing(missingMap.get(field));
    return config.format(fieldType.docValueFormat(format, null));
}