Examples with MappedFieldType org.elasticsearch.index.mapper.MappedFieldType used on opensource projects

Example 96 with MappedFieldType

use of org.elasticsearch.index.mapper.MappedFieldType in project elasticsearch by elastic.

the class ExpressionScriptEngineService method search.

@Override
public SearchScript search(CompiledScript compiledScript, SearchLookup lookup, @Nullable Map<String, Object> vars) {
    Expression expr = (Expression) compiledScript.compiled();
    MapperService mapper = lookup.doc().mapperService();
    // NOTE: if we need to do anything complicated with bindings in the future, we can just extend Bindings,
    // instead of complicating SimpleBindings (which should stay simple)
    SimpleBindings bindings = new SimpleBindings();
    ReplaceableConstDoubleValueSource specialValue = null;
    boolean needsScores = false;
    for (String variable : expr.variables) {
        try {
            if (variable.equals("_score")) {
                bindings.add(new SortField("_score", SortField.Type.SCORE));
                needsScores = true;
            } else if (variable.equals("_value")) {
                specialValue = new ReplaceableConstDoubleValueSource();
                bindings.add("_value", specialValue);
            // noop: _value is special for aggregations, and is handled in ExpressionScriptBindings
            // TODO: if some uses it in a scoring expression, they will get a nasty failure when evaluating...need a
            // way to know this is for aggregations and so _value is ok to have...
            } else if (vars != null && vars.containsKey(variable)) {
                // TODO: document and/or error if vars contains _score?
                // NOTE: by checking for the variable in vars first, it allows masking document fields with a global constant,
                // but if we were to reverse it, we could provide a way to supply dynamic defaults for documents missing the field?
                Object value = vars.get(variable);
                if (value instanceof Number) {
                    bindings.add(variable, new DoubleConstValueSource(((Number) value).doubleValue()).asDoubleValuesSource());
                } else {
                    throw new ParseException("Parameter [" + variable + "] must be a numeric type", 0);
                }
            } else {
                String fieldname = null;
                String methodname = null;
                // .value is the default for doc['field'], its optional.
                String variablename = "value";
                // true if the variable is of type doc['field'].date.xxx
                boolean dateAccessor = false;
                VariableContext[] parts = VariableContext.parse(variable);
                if (parts[0].text.equals("doc") == false) {
                    throw new ParseException("Unknown variable [" + parts[0].text + "]", 0);
                }
                if (parts.length < 2 || parts[1].type != VariableContext.Type.STR_INDEX) {
                    throw new ParseException("Variable 'doc' must be used with a specific field like: doc['myfield']", 3);
                } else {
                    fieldname = parts[1].text;
                }
                if (parts.length == 3) {
                    if (parts[2].type == VariableContext.Type.METHOD) {
                        methodname = parts[2].text;
                    } else if (parts[2].type == VariableContext.Type.MEMBER) {
                        variablename = parts[2].text;
                    } else {
                        throw new IllegalArgumentException("Only member variables or member methods may be accessed on a field when not accessing the field directly");
                    }
                }
                if (parts.length > 3) {
                    // access to the .date "object" within the field
                    if (parts.length == 4 && ("date".equals(parts[2].text) || "getDate".equals(parts[2].text))) {
                        if (parts[3].type == VariableContext.Type.METHOD) {
                            methodname = parts[3].text;
                            dateAccessor = true;
                        } else if (parts[3].type == VariableContext.Type.MEMBER) {
                            variablename = parts[3].text;
                            dateAccessor = true;
                        }
                    }
                    if (!dateAccessor) {
                        throw new IllegalArgumentException("Variable [" + variable + "] does not follow an allowed format of either doc['field'] or doc['field'].method()");
                    }
                }
                MappedFieldType fieldType = mapper.fullName(fieldname);
                if (fieldType == null) {
                    throw new ParseException("Field [" + fieldname + "] does not exist in mappings", 5);
                }
                IndexFieldData<?> fieldData = lookup.doc().fieldDataService().getForField(fieldType);
                // delegate valuesource creation based on field's type
                // there are three types of "fields" to expressions, and each one has a different "api" of variables and methods.
                final ValueSource valueSource;
                if (fieldType instanceof GeoPointFieldType) {
                    // geo
                    if (methodname == null) {
                        valueSource = GeoField.getVariable(fieldData, fieldname, variablename);
                    } else {
                        valueSource = GeoField.getMethod(fieldData, fieldname, methodname);
                    }
                } else if (fieldType instanceof DateFieldMapper.DateFieldType) {
                    if (dateAccessor) {
                        // date object
                        if (methodname == null) {
                            valueSource = DateObject.getVariable(fieldData, fieldname, variablename);
                        } else {
                            valueSource = DateObject.getMethod(fieldData, fieldname, methodname);
                        }
                    } else {
                        // date field itself
                        if (methodname == null) {
                            valueSource = DateField.getVariable(fieldData, fieldname, variablename);
                        } else {
                            valueSource = DateField.getMethod(fieldData, fieldname, methodname);
                        }
                    }
                } else if (fieldData instanceof IndexNumericFieldData) {
                    // number
                    if (methodname == null) {
                        valueSource = NumericField.getVariable(fieldData, fieldname, variablename);
                    } else {
                        valueSource = NumericField.getMethod(fieldData, fieldname, methodname);
                    }
                } else {
                    throw new ParseException("Field [" + fieldname + "] must be numeric, date, or geopoint", 5);
                }
                needsScores |= valueSource.getSortField(false).needsScores();
                bindings.add(variable, valueSource.asDoubleValuesSource());
            }
        } catch (Exception e) {
            // we defer "binding" of variables until here: give context for that variable
            throw convertToScriptException("link error", expr.sourceText, variable, e);
        }
    }
    return new ExpressionSearchScript(compiledScript, bindings, specialValue, needsScores);
}

Example 97 with MappedFieldType

use of org.elasticsearch.index.mapper.MappedFieldType in project elasticsearch by elastic.

the class IndexShardTests method testSearcherWrapperWorksWithGlobalOrdinals.

public void testSearcherWrapperWorksWithGlobalOrdinals() throws IOException {
    IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {

        @Override
        public DirectoryReader wrap(DirectoryReader reader) throws IOException {
            return new FieldMaskingReader("foo", reader);
        }

        @Override
        public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
            return searcher;
        }
    };
    Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build();
    IndexMetaData metaData = IndexMetaData.builder("test").putMapping("test", "{ \"properties\": { \"foo\":  { \"type\": \"text\", \"fielddata\": true }}}").settings(settings).primaryTerm(0, 1).build();
    IndexShard shard = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, wrapper);
    recoveryShardFromStore(shard);
    indexDoc(shard, "test", "0", "{\"foo\" : \"bar\"}");
    shard.refresh("created segment 1");
    indexDoc(shard, "test", "1", "{\"foobar\" : \"bar\"}");
    shard.refresh("created segment 2");
    // test global ordinals are evicted
    MappedFieldType foo = shard.mapperService().fullName("foo");
    IndexFieldData.Global ifd = shard.indexFieldDataService().getForField(foo);
    FieldDataStats before = shard.fieldData().stats("foo");
    assertThat(before.getMemorySizeInBytes(), equalTo(0L));
    FieldDataStats after = null;
    try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
        assertThat("we have to have more than one segment", searcher.getDirectoryReader().leaves().size(), greaterThan(1));
        ifd.loadGlobal(searcher.getDirectoryReader());
        after = shard.fieldData().stats("foo");
        assertEquals(after.getEvictions(), before.getEvictions());
        // If a field doesn't exist an empty IndexFieldData is returned and that isn't cached:
        assertThat(after.getMemorySizeInBytes(), equalTo(0L));
    }
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), after.getMemorySizeInBytes());
    shard.flush(new FlushRequest().force(true).waitIfOngoing(true));
    shard.refresh("test");
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), before.getMemorySizeInBytes());
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    closeShards(shard);
}

Example 98 with MappedFieldType

use of org.elasticsearch.index.mapper.MappedFieldType in project elasticsearch by elastic.

the class IpRangeAggregatorTests method testRanges.

public void testRanges() throws Exception {
    boolean v4 = randomBoolean();
    IpRangeAggregationBuilder builder = new IpRangeAggregationBuilder("test_agg").field("field");
    int numRanges = randomIntBetween(1, 10);
    Tuple<BytesRef, BytesRef>[] requestedRanges = new Tuple[numRanges];
    for (int i = 0; i < numRanges; i++) {
        Tuple<InetAddress, BytesRef>[] arr = new Tuple[2];
        for (int j = 0; j < 2; j++) {
            InetAddress addr = randomIp(v4);
            arr[j] = new Tuple(addr, new BytesRef(InetAddressPoint.encode(addr)));
        }
        Arrays.sort(arr, (t1, t2) -> t1.v2().compareTo(t2.v2()));
        if (rarely()) {
            if (randomBoolean()) {
                builder.addRange(NetworkAddress.format(arr[0].v1()), null);
                requestedRanges[i] = new Tuple(arr[0].v2(), null);
            } else {
                builder.addRange(null, NetworkAddress.format(arr[1].v1()));
                requestedRanges[i] = new Tuple(null, arr[1].v2());
            }
        } else {
            builder.addRange(NetworkAddress.format(arr[0].v1()), NetworkAddress.format(arr[1].v1()));
            requestedRanges[i] = new Tuple(arr[0].v2(), arr[1].v2());
        }
    }
    Arrays.sort(requestedRanges, RANGE_COMPARATOR);
    int[] expectedCounts = new int[numRanges];
    try (Directory dir = newDirectory();
        RandomIndexWriter w = new RandomIndexWriter(random(), dir)) {
        int numDocs = randomIntBetween(10, 100);
        for (int i = 0; i < numDocs; i++) {
            Document doc = new Document();
            int numValues = randomIntBetween(1, 5);
            BytesRef[] values = new BytesRef[numValues];
            for (int j = 0; j < numValues; j++) {
                values[j] = new BytesRef(InetAddressPoint.encode(randomIp(v4)));
                doc.add(new SortedSetDocValuesField("field", values[j]));
            }
            Arrays.sort(values);
            for (int j = 0; j < numRanges; j++) {
                for (int k = 0; k < numValues; k++) {
                    if (isInRange(values[k], requestedRanges[j].v1(), requestedRanges[j].v2())) {
                        expectedCounts[j]++;
                        break;
                    }
                }
            }
            w.addDocument(doc);
        }
        MappedFieldType fieldType = new IpFieldMapper.IpFieldType();
        fieldType.setName("field");
        try (IndexReader reader = w.getReader()) {
            IndexSearcher searcher = new IndexSearcher(reader);
            InternalBinaryRange range = search(searcher, new MatchAllDocsQuery(), builder, fieldType);
            assertEquals(numRanges, range.getBuckets().size());
            for (int i = 0; i < range.getBuckets().size(); i++) {
                Tuple<BytesRef, BytesRef> expected = requestedRanges[i];
                Range.Bucket bucket = range.getBuckets().get(i);
                if (expected.v1() == null) {
                    assertNull(bucket.getFrom());
                } else {
                    assertEquals(DocValueFormat.IP.format(expected.v1()), bucket.getFrom());
                }
                if (expected.v2() == null) {
                    assertNull(bucket.getTo());
                } else {
                    assertEquals(DocValueFormat.IP.format(expected.v2()), bucket.getTo());
                }
                assertEquals(expectedCounts[i], bucket.getDocCount());
            }
        }
    }
}

Example 99 with MappedFieldType

use of org.elasticsearch.index.mapper.MappedFieldType in project elasticsearch by elastic.

the class DiversifiedSamplerTests method testDiversifiedSampler_noDocs.

public void testDiversifiedSampler_noDocs() throws Exception {
    Directory directory = newDirectory();
    RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory);
    indexWriter.close();
    IndexReader indexReader = DirectoryReader.open(directory);
    IndexSearcher indexSearcher = new IndexSearcher(indexReader);
    MappedFieldType idFieldType = new KeywordFieldMapper.KeywordFieldType();
    idFieldType.setName("id");
    idFieldType.setHasDocValues(true);
    MappedFieldType genreFieldType = new KeywordFieldMapper.KeywordFieldType();
    genreFieldType.setName("genre");
    genreFieldType.setHasDocValues(true);
    DiversifiedAggregationBuilder builder = new DiversifiedAggregationBuilder("_name").field(genreFieldType.name()).subAggregation(new TermsAggregationBuilder("terms", null).field("id"));
    InternalSampler result = search(indexSearcher, new MatchAllDocsQuery(), builder, genreFieldType, idFieldType);
    Terms terms = result.getAggregations().get("terms");
    assertEquals(0, terms.getBuckets().size());
    indexReader.close();
    directory.close();
}

Example 100 with MappedFieldType

use of org.elasticsearch.index.mapper.MappedFieldType in project elasticsearch by elastic.

the class SamplerAggregatorTests method testSampler.

/**
     * Uses the sampler aggregation to find the minimum value of a field out of the top 3 scoring documents in a search.
     */
public void testSampler() throws IOException {
    TextFieldType textFieldType = new TextFieldType();
    textFieldType.setIndexAnalyzer(new NamedAnalyzer("foo", AnalyzerScope.GLOBAL, new StandardAnalyzer()));
    MappedFieldType numericFieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG);
    numericFieldType.setName("int");
    IndexWriterConfig indexWriterConfig = newIndexWriterConfig();
    indexWriterConfig.setMaxBufferedDocs(100);
    // flush on open to have a single segment with predictable docIds
    indexWriterConfig.setRAMBufferSizeMB(100);
    try (Directory dir = newDirectory();
        IndexWriter w = new IndexWriter(dir, indexWriterConfig)) {
        for (long value : new long[] { 7, 3, -10, -6, 5, 50 }) {
            Document doc = new Document();
            StringBuilder text = new StringBuilder();
            for (int i = 0; i < value; i++) {
                text.append("good ");
            }
            doc.add(new Field("text", text.toString(), textFieldType));
            doc.add(new SortedNumericDocValuesField("int", value));
            w.addDocument(doc);
        }
        SamplerAggregationBuilder aggBuilder = new SamplerAggregationBuilder("sampler").shardSize(3).subAggregation(new MinAggregationBuilder("min").field("int"));
        try (IndexReader reader = DirectoryReader.open(w)) {
            assertEquals("test expects a single segment", 1, reader.leaves().size());
            IndexSearcher searcher = new IndexSearcher(reader);
            Sampler sampler = searchAndReduce(searcher, new TermQuery(new Term("text", "good")), aggBuilder, textFieldType, numericFieldType);
            Min min = sampler.getAggregations().get("min");
            assertEquals(5.0, min.getValue(), 0);
        }
    }
}