Examples with BitsetFilterCache org.opensearch.index.cache.bitset.BitsetFilterCache used on opensource projects

Example 1 with BitsetFilterCache

use of org.opensearch.index.cache.bitset.BitsetFilterCache in project OpenSearch by opensearch-project.

the class ContextIndexSearcherTests method doTestContextIndexSearcher.

public void doTestContextIndexSearcher(boolean sparse, boolean deletions) throws IOException {
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig(null));
    Document doc = new Document();
    StringField allowedField = new StringField("allowed", "yes", Field.Store.NO);
    doc.add(allowedField);
    StringField fooField = new StringField("foo", "bar", Field.Store.NO);
    doc.add(fooField);
    StringField deleteField = new StringField("delete", "no", Field.Store.NO);
    doc.add(deleteField);
    IntPoint pointField = new IntPoint("point", 1, 2);
    doc.add(pointField);
    w.addDocument(doc);
    if (deletions) {
        // add a document that matches foo:bar but will be deleted
        deleteField.setStringValue("yes");
        w.addDocument(doc);
        deleteField.setStringValue("no");
    }
    allowedField.setStringValue("no");
    w.addDocument(doc);
    if (sparse) {
        for (int i = 0; i < 1000; ++i) {
            w.addDocument(doc);
        }
        w.forceMerge(1);
    }
    w.deleteDocuments(new Term("delete", "yes"));
    IndexSettings settings = IndexSettingsModule.newIndexSettings("_index", Settings.EMPTY);
    BitsetFilterCache.Listener listener = new BitsetFilterCache.Listener() {

        @Override
        public void onCache(ShardId shardId, Accountable accountable) {
        }

        @Override
        public void onRemoval(ShardId shardId, Accountable accountable) {
        }
    };
    DirectoryReader reader = OpenSearchDirectoryReader.wrap(DirectoryReader.open(w), new ShardId(settings.getIndex(), 0));
    BitsetFilterCache cache = new BitsetFilterCache(settings, listener);
    Query roleQuery = new TermQuery(new Term("allowed", "yes"));
    BitSet bitSet = cache.getBitSetProducer(roleQuery).getBitSet(reader.leaves().get(0));
    if (sparse) {
        assertThat(bitSet, instanceOf(SparseFixedBitSet.class));
    } else {
        assertThat(bitSet, instanceOf(FixedBitSet.class));
    }
    DocumentSubsetDirectoryReader filteredReader = new DocumentSubsetDirectoryReader(reader, cache, roleQuery);
    ContextIndexSearcher searcher = new ContextIndexSearcher(filteredReader, IndexSearcher.getDefaultSimilarity(), IndexSearcher.getDefaultQueryCache(), IndexSearcher.getDefaultQueryCachingPolicy(), true);
    for (LeafReaderContext context : searcher.getIndexReader().leaves()) {
        assertThat(context.reader(), instanceOf(SequentialStoredFieldsLeafReader.class));
        SequentialStoredFieldsLeafReader lf = (SequentialStoredFieldsLeafReader) context.reader();
        assertNotNull(lf.getSequentialStoredFieldsReader());
    }
    // Assert wrapping
    assertEquals(ExitableDirectoryReader.class, searcher.getIndexReader().getClass());
    for (LeafReaderContext lrc : searcher.getIndexReader().leaves()) {
        assertEquals(ExitableLeafReader.class, lrc.reader().getClass());
        assertNotEquals(ExitableTerms.class, lrc.reader().terms("foo").getClass());
        assertNotEquals(ExitablePointValues.class, lrc.reader().getPointValues("point").getClass());
    }
    searcher.addQueryCancellation(() -> {
    });
    for (LeafReaderContext lrc : searcher.getIndexReader().leaves()) {
        assertEquals(ExitableTerms.class, lrc.reader().terms("foo").getClass());
        assertEquals(ExitablePointValues.class, lrc.reader().getPointValues("point").getClass());
    }
    // Searching a non-existing term will trigger a null scorer
    assertEquals(0, searcher.count(new TermQuery(new Term("non_existing_field", "non_existing_value"))));
    assertEquals(1, searcher.count(new TermQuery(new Term("foo", "bar"))));
    // make sure scorers are created only once, see #1725
    assertEquals(1, searcher.count(new CreateScorerOnceQuery(new MatchAllDocsQuery())));
    TopDocs topDocs = searcher.search(new BoostQuery(new ConstantScoreQuery(new TermQuery(new Term("foo", "bar"))), 3f), 1);
    assertEquals(1, topDocs.totalHits.value);
    assertEquals(1, topDocs.scoreDocs.length);
    assertEquals(3f, topDocs.scoreDocs[0].score, 0);
    IOUtils.close(reader, w, dir);
}

Example 2 with BitsetFilterCache

use of org.opensearch.index.cache.bitset.BitsetFilterCache in project OpenSearch by opensearch-project.

the class AbstractSortTestCase method createMockShardContext.

protected final QueryShardContext createMockShardContext(IndexSearcher searcher) {
    Index index = new Index(randomAlphaOfLengthBetween(1, 10), "_na_");
    IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).build());
    BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, Mockito.mock(BitsetFilterCache.Listener.class));
    TriFunction<MappedFieldType, String, Supplier<SearchLookup>, IndexFieldData<?>> indexFieldDataLookup = (fieldType, fieldIndexName, searchLookup) -> {
        IndexFieldData.Builder builder = fieldType.fielddataBuilder(fieldIndexName, searchLookup);
        return builder.build(new IndexFieldDataCache.None(), null);
    };
    return new QueryShardContext(0, idxSettings, BigArrays.NON_RECYCLING_INSTANCE, bitsetFilterCache, indexFieldDataLookup, null, null, scriptService, xContentRegistry(), namedWriteableRegistry, null, searcher, () -> randomNonNegativeLong(), null, null, () -> true, null) {

        @Override
        public MappedFieldType fieldMapper(String name) {
            return provideMappedFieldType(name);
        }

        @Override
        public ObjectMapper getObjectMapper(String name) {
            BuilderContext context = new BuilderContext(this.getIndexSettings().getSettings(), new ContentPath());
            return new ObjectMapper.Builder<>(name).nested(Nested.newNested()).build(context);
        }
    };
}

Example 3 with BitsetFilterCache

use of org.opensearch.index.cache.bitset.BitsetFilterCache in project OpenSearch by opensearch-project.

the class AggregatorTestCase method createSearchContext.

protected SearchContext createSearchContext(IndexSearcher indexSearcher, IndexSettings indexSettings, Query query, MultiBucketConsumer bucketConsumer, CircuitBreakerService circuitBreakerService, MappedFieldType... fieldTypes) throws IOException {
    QueryCache queryCache = new DisabledQueryCache(indexSettings);
    QueryCachingPolicy queryCachingPolicy = new QueryCachingPolicy() {

        @Override
        public void onUse(Query query) {
        }

        @Override
        public boolean shouldCache(Query query) {
            // never cache a query
            return false;
        }
    };
    ContextIndexSearcher contextIndexSearcher = new ContextIndexSearcher(indexSearcher.getIndexReader(), indexSearcher.getSimilarity(), queryCache, queryCachingPolicy, false);
    SearchContext searchContext = mock(SearchContext.class);
    when(searchContext.numberOfShards()).thenReturn(1);
    when(searchContext.searcher()).thenReturn(contextIndexSearcher);
    when(searchContext.fetchPhase()).thenReturn(new FetchPhase(Arrays.asList(new FetchSourcePhase(), new FetchDocValuesPhase())));
    when(searchContext.bitsetFilterCache()).thenReturn(new BitsetFilterCache(indexSettings, mock(Listener.class)));
    IndexShard indexShard = mock(IndexShard.class);
    when(indexShard.shardId()).thenReturn(new ShardId("test", "test", 0));
    when(searchContext.indexShard()).thenReturn(indexShard);
    when(searchContext.aggregations()).thenReturn(new SearchContextAggregations(AggregatorFactories.EMPTY, bucketConsumer));
    when(searchContext.query()).thenReturn(query);
    /*
         * Always use the circuit breaking big arrays instance so that the CircuitBreakerService
         * we're passed gets a chance to break.
         */
    BigArrays bigArrays = new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), circuitBreakerService).withCircuitBreaking();
    when(searchContext.bigArrays()).thenReturn(bigArrays);
    // TODO: now just needed for top_hits, this will need to be revised for other agg unit tests:
    MapperService mapperService = mapperServiceMock();
    when(mapperService.getIndexSettings()).thenReturn(indexSettings);
    when(mapperService.hasNested()).thenReturn(false);
    when(searchContext.mapperService()).thenReturn(mapperService);
    IndexFieldDataService ifds = new IndexFieldDataService(indexSettings, new IndicesFieldDataCache(Settings.EMPTY, new IndexFieldDataCache.Listener() {
    }), circuitBreakerService, mapperService);
    QueryShardContext queryShardContext = queryShardContextMock(contextIndexSearcher, mapperService, indexSettings, circuitBreakerService, bigArrays);
    when(searchContext.getQueryShardContext()).thenReturn(queryShardContext);
    when(queryShardContext.getObjectMapper(anyString())).thenAnswer(invocation -> {
        String fieldName = (String) invocation.getArguments()[0];
        if (fieldName.startsWith(NESTEDFIELD_PREFIX)) {
            BuilderContext context = new BuilderContext(indexSettings.getSettings(), new ContentPath());
            return new ObjectMapper.Builder<>(fieldName).nested(Nested.newNested()).build(context);
        }
        return null;
    });
    Map<String, MappedFieldType> fieldNameToType = new HashMap<>();
    fieldNameToType.putAll(Arrays.stream(fieldTypes).filter(Objects::nonNull).collect(Collectors.toMap(MappedFieldType::name, Function.identity())));
    fieldNameToType.putAll(getFieldAliases(fieldTypes));
    registerFieldTypes(searchContext, mapperService, fieldNameToType);
    doAnswer(invocation -> {
        /* Store the release-ables so we can release them at the end of the test case. This is important because aggregations don't
             * close their sub-aggregations. This is fairly similar to what the production code does. */
        releasables.add((Releasable) invocation.getArguments()[0]);
        return null;
    }).when(searchContext).addReleasable(any());
    return searchContext;
}