Examples with IndexFieldData org.elasticsearch.index.fielddata.IndexFieldData used on opensource projects

Example 1 with IndexFieldData

use of org.elasticsearch.index.fielddata.IndexFieldData in project elasticsearch by elastic.

the class SliceBuilder method toFilter.

public Query toFilter(QueryShardContext context, int shardId, int numShards) {
    final MappedFieldType type = context.fieldMapper(field);
    if (type == null) {
        throw new IllegalArgumentException("field " + field + " not found");
    }
    boolean useTermQuery = false;
    if (UidFieldMapper.NAME.equals(field)) {
        useTermQuery = true;
    } else if (type.hasDocValues() == false) {
        throw new IllegalArgumentException("cannot load numeric doc values on " + field);
    } else {
        IndexFieldData ifm = context.getForField(type);
        if (ifm instanceof IndexNumericFieldData == false) {
            throw new IllegalArgumentException("cannot load numeric doc values on " + field);
        }
    }
    if (numShards == 1) {
        return useTermQuery ? new TermsSliceQuery(field, id, max) : new DocValuesSliceQuery(field, id, max);
    }
    if (max >= numShards) {
        // the number of slices is greater than the number of shards
        // in such case we can reduce the number of requested shards by slice
        // first we check if the slice is responsible of this shard
        int targetShard = id % numShards;
        if (targetShard != shardId) {
            // the shard is not part of this slice, we can skip it.
            return new MatchNoDocsQuery("this shard is not part of the slice");
        }
        // compute the number of slices where this shard appears
        int numSlicesInShard = max / numShards;
        int rest = max % numShards;
        if (rest > targetShard) {
            numSlicesInShard++;
        }
        if (numSlicesInShard == 1) {
            // this shard has only one slice so we must check all the documents
            return new MatchAllDocsQuery();
        }
        // get the new slice id for this shard
        int shardSlice = id / numShards;
        return useTermQuery ? new TermsSliceQuery(field, shardSlice, numSlicesInShard) : new DocValuesSliceQuery(field, shardSlice, numSlicesInShard);
    }
    // the number of shards is greater than the number of slices
    // check if the shard is assigned to the slice
    int targetSlice = shardId % max;
    if (id != targetSlice) {
        // the shard is not part of this slice, we can skip it.
        return new MatchNoDocsQuery("this shard is not part of the slice");
    }
    return new MatchAllDocsQuery();
}

Example 2 with IndexFieldData

use of org.elasticsearch.index.fielddata.IndexFieldData in project elasticsearch by elastic.

the class FieldSortBuilder method build.

@Override
public SortFieldAndFormat build(QueryShardContext context) throws IOException {
    if (DOC_FIELD_NAME.equals(fieldName)) {
        if (order == SortOrder.DESC) {
            return SORT_DOC_REVERSE;
        } else {
            return SORT_DOC;
        }
    } else {
        MappedFieldType fieldType = context.fieldMapper(fieldName);
        if (fieldType == null) {
            if (unmappedType != null) {
                fieldType = context.getMapperService().unmappedFieldType(unmappedType);
            } else {
                throw new QueryShardException(context, "No mapping found for [" + fieldName + "] in order to sort on");
            }
        }
        MultiValueMode localSortMode = null;
        if (sortMode != null) {
            localSortMode = MultiValueMode.fromString(sortMode.toString());
        }
        boolean reverse = (order == SortOrder.DESC);
        if (localSortMode == null) {
            localSortMode = reverse ? MultiValueMode.MAX : MultiValueMode.MIN;
        }
        final Nested nested = resolveNested(context, nestedPath, nestedFilter);
        IndexFieldData<?> fieldData = context.getForField(fieldType);
        if (fieldData instanceof IndexNumericFieldData == false && (sortMode == SortMode.SUM || sortMode == SortMode.AVG || sortMode == SortMode.MEDIAN)) {
            throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields");
        }
        IndexFieldData.XFieldComparatorSource fieldComparatorSource = fieldData.comparatorSource(missing, localSortMode, nested);
        SortField field = new SortField(fieldType.name(), fieldComparatorSource, reverse);
        return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null));
    }
}

Example 3 with IndexFieldData

use of org.elasticsearch.index.fielddata.IndexFieldData in project crate by crate.

the class LuceneQueryBuilderTest method prepare.

@Before
public void prepare() throws Exception {
    DocTableInfo users = TestingTableInfo.builder(new TableIdent(null, "users"), null).add("name", DataTypes.STRING).add("x", DataTypes.INTEGER).add("d", DataTypes.DOUBLE).add("d_array", new ArrayType(DataTypes.DOUBLE)).add("y_array", new ArrayType(DataTypes.LONG)).add("shape", DataTypes.GEO_SHAPE).add("point", DataTypes.GEO_POINT).build();
    TableRelation usersTr = new TableRelation(users);
    sources = ImmutableMap.of(new QualifiedName("users"), usersTr);
    expressions = new SqlExpressions(sources, usersTr);
    builder = new LuceneQueryBuilder(expressions.getInstance(Functions.class));
    indexCache = mock(IndexCache.class, Answers.RETURNS_MOCKS.get());
    Path tempDir = createTempDir();
    Settings indexSettings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put("path.home", tempDir).build();
    Index index = new Index(users.ident().indexName());
    when(indexCache.indexSettings()).thenReturn(indexSettings);
    AnalysisService analysisService = createAnalysisService(indexSettings, index);
    mapperService = createMapperService(index, indexSettings, analysisService);
    // @formatter:off
    XContentBuilder xContentBuilder = XContentFactory.jsonBuilder().startObject().startObject("default").startObject("properties").startObject("name").field("type", "string").endObject().startObject("x").field("type", "integer").endObject().startObject("d").field("type", "double").endObject().startObject("point").field("type", "geo_point").endObject().startObject("shape").field("type", "geo_shape").endObject().startObject("d_array").field("type", "array").startObject("inner").field("type", "double").endObject().endObject().startObject("y_array").field("type", "array").startObject("inner").field("type", "integer").endObject().endObject().endObject().endObject().endObject();
    // @formatter:on
    mapperService.merge("default", new CompressedXContent(xContentBuilder.bytes()), MapperService.MergeReason.MAPPING_UPDATE, true);
    indexFieldDataService = mock(IndexFieldDataService.class);
    IndexFieldData geoFieldData = mock(IndexGeoPointFieldData.class);
    when(geoFieldData.getFieldNames()).thenReturn(new MappedFieldType.Names("point"));
    when(indexFieldDataService.getForField(mapperService.smartNameFieldType("point"))).thenReturn(geoFieldData);
}

Example 4 with IndexFieldData

use of org.elasticsearch.index.fielddata.IndexFieldData in project elasticsearch by elastic.

the class IndexShardTests method testSearcherWrapperWorksWithGlobalOrdinals.

public void testSearcherWrapperWorksWithGlobalOrdinals() throws IOException {
    IndexSearcherWrapper wrapper = new IndexSearcherWrapper() {

        @Override
        public DirectoryReader wrap(DirectoryReader reader) throws IOException {
            return new FieldMaskingReader("foo", reader);
        }

        @Override
        public IndexSearcher wrap(IndexSearcher searcher) throws EngineException {
            return searcher;
        }
    };
    Settings settings = Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetaData.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetaData.SETTING_NUMBER_OF_SHARDS, 1).build();
    IndexMetaData metaData = IndexMetaData.builder("test").putMapping("test", "{ \"properties\": { \"foo\":  { \"type\": \"text\", \"fielddata\": true }}}").settings(settings).primaryTerm(0, 1).build();
    IndexShard shard = newShard(new ShardId(metaData.getIndex(), 0), true, "n1", metaData, wrapper);
    recoveryShardFromStore(shard);
    indexDoc(shard, "test", "0", "{\"foo\" : \"bar\"}");
    shard.refresh("created segment 1");
    indexDoc(shard, "test", "1", "{\"foobar\" : \"bar\"}");
    shard.refresh("created segment 2");
    // test global ordinals are evicted
    MappedFieldType foo = shard.mapperService().fullName("foo");
    IndexFieldData.Global ifd = shard.indexFieldDataService().getForField(foo);
    FieldDataStats before = shard.fieldData().stats("foo");
    assertThat(before.getMemorySizeInBytes(), equalTo(0L));
    FieldDataStats after = null;
    try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
        assertThat("we have to have more than one segment", searcher.getDirectoryReader().leaves().size(), greaterThan(1));
        ifd.loadGlobal(searcher.getDirectoryReader());
        after = shard.fieldData().stats("foo");
        assertEquals(after.getEvictions(), before.getEvictions());
        // If a field doesn't exist an empty IndexFieldData is returned and that isn't cached:
        assertThat(after.getMemorySizeInBytes(), equalTo(0L));
    }
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), after.getMemorySizeInBytes());
    shard.flush(new FlushRequest().force(true).waitIfOngoing(true));
    shard.refresh("test");
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), before.getMemorySizeInBytes());
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    closeShards(shard);
}