Examples with IndexFieldDataService org.opensearch.index.fielddata.IndexFieldDataService used on opensource projects

Example 1 with IndexFieldDataService

use of org.opensearch.index.fielddata.IndexFieldDataService in project OpenSearch by opensearch-project.

the class IndexShardTests method testReaderWrapperWorksWithGlobalOrdinals.

public void testReaderWrapperWorksWithGlobalOrdinals() throws IOException {
    CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = reader -> new FieldMaskingReader("foo", reader);
    Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
    IndexMetadata metadata = IndexMetadata.builder("test").putMapping("_doc", "{ \"properties\": { \"foo\":  { \"type\": \"text\", \"fielddata\": true }}}").settings(settings).primaryTerm(0, 1).build();
    IndexShard shard = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, wrapper);
    recoverShardFromStore(shard);
    indexDoc(shard, "_doc", "0", "{\"foo\" : \"bar\"}");
    shard.refresh("created segment 1");
    indexDoc(shard, "_doc", "1", "{\"foobar\" : \"bar\"}");
    shard.refresh("created segment 2");
    // test global ordinals are evicted
    MappedFieldType foo = shard.mapperService().fieldType("foo");
    IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(shard.indexSettings.getNodeSettings(), new IndexFieldDataCache.Listener() {
    });
    IndexFieldDataService indexFieldDataService = new IndexFieldDataService(shard.indexSettings, indicesFieldDataCache, new NoneCircuitBreakerService(), shard.mapperService());
    IndexFieldData.Global ifd = indexFieldDataService.getForField(foo, "test", () -> {
        throw new UnsupportedOperationException("search lookup not available");
    });
    FieldDataStats before = shard.fieldData().stats("foo");
    assertThat(before.getMemorySizeInBytes(), equalTo(0L));
    FieldDataStats after = null;
    try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
        assertThat("we have to have more than one segment", searcher.getDirectoryReader().leaves().size(), greaterThan(1));
        ifd.loadGlobal(searcher.getDirectoryReader());
        after = shard.fieldData().stats("foo");
        assertEquals(after.getEvictions(), before.getEvictions());
        // If a field doesn't exist an empty IndexFieldData is returned and that isn't cached:
        assertThat(after.getMemorySizeInBytes(), equalTo(0L));
    }
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), after.getMemorySizeInBytes());
    shard.flush(new FlushRequest().force(true).waitIfOngoing(true));
    shard.refresh("test");
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), before.getMemorySizeInBytes());
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    closeShards(shard);
}

Example 2 with IndexFieldDataService

use of org.opensearch.index.fielddata.IndexFieldDataService in project OpenSearch by opensearch-project.

the class AggregatorTestCase method createSearchContext.

protected SearchContext createSearchContext(IndexSearcher indexSearcher, IndexSettings indexSettings, Query query, MultiBucketConsumer bucketConsumer, CircuitBreakerService circuitBreakerService, MappedFieldType... fieldTypes) throws IOException {
    QueryCache queryCache = new DisabledQueryCache(indexSettings);
    QueryCachingPolicy queryCachingPolicy = new QueryCachingPolicy() {

        @Override
        public void onUse(Query query) {
        }

        @Override
        public boolean shouldCache(Query query) {
            // never cache a query
            return false;
        }
    };
    ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexSearcher.getIndexReader(), indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, false);
    SearchContext searchContext = mock(SearchContext.class);
    when(searchContext.numberOfShards()).thenReturn(1);
    when(searchContext.searcher()).thenReturn(contextIndexSearcher);
    when(searchContext.fetchPhase()).thenReturn(new FetchPhase(Arrays.asList(new FetchSourcePhase(), new FetchDocValuesPhase())));
    when(searchContext.bitsetFilterCache()).thenReturn(new BitsetFilterCache(indexSettings, mock(Listener.class)));
    IndexShard indexShard = mock(IndexShard.class);
    when(indexShard.shardId()).thenReturn(new ShardId("test", "test", 0));
    when(searchContext.indexShard()).thenReturn(indexShard);
    when(searchContext.aggregations()).thenReturn(new SearchContextAggregations(AggregatorFactories.EMPTY, bucketConsumer));
    when(searchContext.query()).thenReturn(query);
    /*
         * Always use the circuit breaking big arrays instance so that the CircuitBreakerService
         * we're passed gets a chance to break.
         */
    BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), circuitBreakerService).withCircuitBreaking();
    when(searchContext.bigArrays()).thenReturn(bigArrays);
    // TODO: now just needed for top_hits, this will need to be revised for other agg unit tests:
    MapperService mapperService = mapperServiceMock();
    when(mapperService.getIndexSettings()).thenReturn(indexSettings);
    when(mapperService.hasNested()).thenReturn(false);
    when(searchContext.mapperService()).thenReturn(mapperService);
    IndexFieldDataService ifds = new IndexFieldDataService(indexSettings, new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() {
    }), circuitBreakerService, mapperService);
    QueryShardContext queryShardContext = queryShardContextMock(contextIndexSearcher, mapperService, indexSettings, circuitBreakerService, bigArrays);
    when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
    when(queryShardContext.getObjectMapper(anyString())).thenAnswer(invocation -> {
        String fieldName = (String) invocation.getArguments()[0];
        if (fieldName.startsWith(NESTEDFIELD_PREFIX)) {
            BuilderContext context = new BuilderContext(indexSettings.getSettings(), new ContentPath());
            return new ObjectMapper.Builder<>(fieldName).nested(Nested.newNested()).build(context);
        }
        return null;
    });
    Map<String, MappedFieldType> fieldNameToType = new HashMap<>();
    fieldNameToType.putAll(Arrays.stream(fieldTypes).filter(Objects::nonNull).collect(Collectors.toMap(MappedFieldType::name, Function.identity())));
    fieldNameToType.putAll(getFieldAliases(fieldTypes));
    registerFieldTypes(searchContext, mapperService, fieldNameToType);
    doAnswer(invocation -> {
        /* Store the release-ables so we can release them at the end of the test case. This is important because aggregations don't
             * close their sub-aggregations. This is fairly similar to what the production code does. */
        releasables.add((Releasable) invocation.getArguments()[0]);
        return null;
    }).when(searchContext).addReleasable(any());
    return searchContext;
}