Examples with IndexFieldData org.opensearch.index.fielddata.IndexFieldData used on opensource projects

Example 1 with IndexFieldData

use of org.opensearch.index.fielddata.IndexFieldData in project OpenSearch by opensearch-project.

the class IndexShardTests method testReaderWrapperWorksWithGlobalOrdinals.

public void testReaderWrapperWorksWithGlobalOrdinals() throws IOException {
    CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = reader -> new FieldMaskingReader("foo", reader);
    Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
    IndexMetadata metadata = IndexMetadata.builder("test").putMapping("_doc", "{ \"properties\": { \"foo\":  { \"type\": \"text\", \"fielddata\": true }}}").settings(settings).primaryTerm(0, 1).build();
    IndexShard shard = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, wrapper);
    recoverShardFromStore(shard);
    indexDoc(shard, "_doc", "0", "{\"foo\" : \"bar\"}");
    shard.refresh("created segment 1");
    indexDoc(shard, "_doc", "1", "{\"foobar\" : \"bar\"}");
    shard.refresh("created segment 2");
    // test global ordinals are evicted
    MappedFieldType foo = shard.mapperService().fieldType("foo");
    IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(shard.indexSettings.getNodeSettings(), new IndexFieldDataCache.Listener() {
    });
    IndexFieldDataService indexFieldDataService = new IndexFieldDataService(shard.indexSettings, indicesFieldDataCache, new NoneCircuitBreakerService(), shard.mapperService());
    IndexFieldData.Global ifd = indexFieldDataService.getForField(foo, "test", () -> {
        throw new UnsupportedOperationException("search lookup not available");
    });
    FieldDataStats before = shard.fieldData().stats("foo");
    assertThat(before.getMemorySizeInBytes(), equalTo(0L));
    FieldDataStats after = null;
    try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
        assertThat("we have to have more than one segment", searcher.getDirectoryReader().leaves().size(), greaterThan(1));
        ifd.loadGlobal(searcher.getDirectoryReader());
        after = shard.fieldData().stats("foo");
        assertEquals(after.getEvictions(), before.getEvictions());
        // If a field doesn't exist an empty IndexFieldData is returned and that isn't cached:
        assertThat(after.getMemorySizeInBytes(), equalTo(0L));
    }
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), after.getMemorySizeInBytes());
    shard.flush(new FlushRequest().force(true).waitIfOngoing(true));
    shard.refresh("test");
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), before.getMemorySizeInBytes());
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    closeShards(shard);
}

Example 2 with IndexFieldData

use of org.opensearch.index.fielddata.IndexFieldData in project OpenSearch by opensearch-project.

the class SliceBuilder method toFilter.

/**
 * Converts this QueryBuilder to a lucene {@link Query}.
 *
 * @param context Additional information needed to build the query
 */
public Query toFilter(ClusterService clusterService, ShardSearchRequest request, QueryShardContext context, Version minNodeVersion) {
    final MappedFieldType type = context.fieldMapper(field);
    if (type == null) {
        throw new IllegalArgumentException("field " + field + " not found");
    }
    int shardId = request.shardId().id();
    int numShards = context.getIndexSettings().getNumberOfShards();
    if ((request.preference() != null || request.indexRoutings().length > 0)) {
        GroupShardsIterator<ShardIterator> group = buildShardIterator(clusterService, request);
        assert group.size() <= numShards : "index routing shards: " + group.size() + " cannot be greater than total number of shards: " + numShards;
        if (group.size() < numShards) {
            /*
                 * The routing of this request targets a subset of the shards of this index so we need to we retrieve
                 * the original {@link GroupShardsIterator} and compute the request shard id and number of
                 * shards from it.
                 */
            numShards = group.size();
            int ord = 0;
            shardId = -1;
            // remap the original shard id with its index (position) in the sorted shard iterator.
            for (ShardIterator it : group) {
                assert it.shardId().getIndex().equals(request.shardId().getIndex());
                if (request.shardId().equals(it.shardId())) {
                    shardId = ord;
                    break;
                }
                ++ord;
            }
            assert shardId != -1 : "shard id: " + request.shardId().getId() + " not found in index shard routing";
        }
    }
    String field = this.field;
    boolean useTermQuery = false;
    if ("_uid".equals(field)) {
        // on new indices, the _id acts as a _uid
        field = IdFieldMapper.NAME;
        if (context.getIndexSettings().getIndexVersionCreated().onOrAfter(LegacyESVersion.V_7_0_0)) {
            throw new IllegalArgumentException("Computing slices on the [_uid] field is illegal for 7.x indices, use [_id] instead");
        }
        DEPRECATION_LOG.deprecate("slice_on_uid", "Computing slices on the [_uid] field is deprecated for 6.x indices, use [_id] instead");
        useTermQuery = true;
    } else if (IdFieldMapper.NAME.equals(field)) {
        useTermQuery = true;
    } else if (type.hasDocValues() == false) {
        throw new IllegalArgumentException("cannot load numeric doc values on " + field);
    } else {
        IndexFieldData ifm = context.getForField(type);
        if (ifm instanceof IndexNumericFieldData == false) {
            throw new IllegalArgumentException("cannot load numeric doc values on " + field);
        }
    }
    if (numShards == 1) {
        return useTermQuery ? new TermsSliceQuery(field, id, max) : new DocValuesSliceQuery(field, id, max);
    }
    if (max >= numShards) {
        // the number of slices is greater than the number of shards
        // in such case we can reduce the number of requested shards by slice
        // first we check if the slice is responsible of this shard
        int targetShard = id % numShards;
        if (targetShard != shardId) {
            // the shard is not part of this slice, we can skip it.
            return new MatchNoDocsQuery("this shard is not part of the slice");
        }
        // compute the number of slices where this shard appears
        int numSlicesInShard = max / numShards;
        int rest = max % numShards;
        if (rest > targetShard) {
            numSlicesInShard++;
        }
        if (numSlicesInShard == 1) {
            // this shard has only one slice so we must check all the documents
            return new MatchAllDocsQuery();
        }
        // get the new slice id for this shard
        int shardSlice = id / numShards;
        return useTermQuery ? new TermsSliceQuery(field, shardSlice, numSlicesInShard) : new DocValuesSliceQuery(field, shardSlice, numSlicesInShard);
    }
    // the number of shards is greater than the number of slices
    // check if the shard is assigned to the slice
    int targetSlice = shardId % max;
    if (id != targetSlice) {
        // the shard is not part of this slice, we can skip it.
        return new MatchNoDocsQuery("this shard is not part of the slice");
    }
    return new MatchAllDocsQuery();
}

Example 3 with IndexFieldData

use of org.opensearch.index.fielddata.IndexFieldData in project OpenSearch by opensearch-project.

the class LeafDocLookupTests method setUp.

@Before
public void setUp() throws Exception {
    super.setUp();
    MappedFieldType fieldType = mock(MappedFieldType.class);
    when(fieldType.name()).thenReturn("field");
    when(fieldType.valueForDisplay(any())).then(returnsFirstArg());
    MapperService mapperService = mock(MapperService.class);
    when(mapperService.fieldType("field")).thenReturn(fieldType);
    when(mapperService.fieldType("alias")).thenReturn(fieldType);
    docValues = mock(ScriptDocValues.class);
    IndexFieldData<?> fieldData = createFieldData(docValues);
    docLookup = new LeafDocLookup(mapperService, ignored -> fieldData, null);
}

Example 4 with IndexFieldData

use of org.opensearch.index.fielddata.IndexFieldData in project OpenSearch by opensearch-project.

the class PercolateQueryBuilder method wrap.

static QueryShardContext wrap(QueryShardContext shardContext) {
    return new QueryShardContext(shardContext) {

        @Override
        public IndexReader getIndexReader() {
            // the reader of the MemoryIndex. We just use `null` for simplicity.
            return null;
        }

        @Override
        public BitSetProducer bitsetFilter(Query query) {
            return context -> {
                final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(context);
                final IndexSearcher searcher = new IndexSearcher(topLevelContext);
                searcher.setQueryCache(null);
                final Weight weight = searcher.createWeight(searcher.rewrite(query), ScoreMode.COMPLETE_NO_SCORES, 1f);
                final Scorer s = weight.scorer(context);
                if (s != null) {
                    return new BitDocIdSet(BitSet.of(s.iterator(), context.reader().maxDoc())).bits();
                } else {
                    return null;
                }
            };
        }

        @Override
        @SuppressWarnings("unchecked")
        public <IFD extends IndexFieldData<?>> IFD getForField(MappedFieldType fieldType) {
            IndexFieldData.Builder builder = fieldType.fielddataBuilder(shardContext.getFullyQualifiedIndex().getName(), shardContext::lookup);
            IndexFieldDataCache cache = new IndexFieldDataCache.None();
            CircuitBreakerService circuitBreaker = new NoneCircuitBreakerService();
            return (IFD) builder.build(cache, circuitBreaker);
        }
    };
}

Example 5 with IndexFieldData

use of org.opensearch.index.fielddata.IndexFieldData in project OpenSearch by opensearch-project.

the class AbstractSortTestCase method createMockShardContext.

protected final QueryShardContext createMockShardContext(IndexSearcher searcher) {
    Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
    IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build());
    BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class));
    TriFunction<MappedFieldType, String, Supplier<SearchLookup>, IndexFieldData<?>> indexFieldDataLookup = (fieldType, fieldIndexName, searchLookup) -> {
        IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup);
        return builder.build(new IndexFieldDataCache.None(), null);
    };
    return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, () -> randomNonNegativeLong(), null, null, () -> true, null) {

        @Override
        public MappedFieldType fieldMapper(String name) {
            return provideMappedFieldType(name);
        }

        @Override
        public ObjectMapper getObjectMapper(String name) {
            BuilderContext context = new BuilderContext(this.getIndexSettings().getSettings(), new ContentPath());
            return new ObjectMapper.Builder<>(name).nested(Nested.newNested()).build(context);
        }
    };
}