Examples with FieldMaskingReader org.opensearch.test.FieldMaskingReader used on opensource projects

Example 1 with FieldMaskingReader

use of org.opensearch.test.FieldMaskingReader in project OpenSearch by opensearch-project.

the class FieldDataCacheTests method testLoadGlobal_neverCacheIfFieldIsMissing.

public void testLoadGlobal_neverCacheIfFieldIsMissing() throws Exception {
    Directory dir = newDirectory();
    IndexWriterConfig iwc = new IndexWriterConfig(null);
    iwc.setMergePolicy(NoMergePolicy.INSTANCE);
    IndexWriter iw = new IndexWriter(dir, iwc);
    long numDocs = scaledRandomIntBetween(32, 128);
    for (int i = 1; i <= numDocs; i++) {
        Document doc = new Document();
        doc.add(new SortedSetDocValuesField("field1", new BytesRef(String.valueOf(i))));
        doc.add(new StringField("field2", String.valueOf(i), Field.Store.NO));
        iw.addDocument(doc);
        if (i % 24 == 0) {
            iw.commit();
        }
    }
    iw.close();
    DirectoryReader ir = OpenSearchDirectoryReader.wrap(DirectoryReader.open(dir), new ShardId("_index", "_na_", 0));
    DummyAccountingFieldDataCache fieldDataCache = new DummyAccountingFieldDataCache();
    // Testing SortedSetOrdinalsIndexFieldData:
    SortedSetOrdinalsIndexFieldData sortedSetOrdinalsIndexFieldData = createSortedDV("field1", fieldDataCache);
    sortedSetOrdinalsIndexFieldData.loadGlobal(ir);
    assertThat(fieldDataCache.cachedGlobally, equalTo(1));
    sortedSetOrdinalsIndexFieldData.loadGlobal(new FieldMaskingReader("field1", ir));
    assertThat(fieldDataCache.cachedGlobally, equalTo(1));
    // Testing PagedBytesIndexFieldData
    PagedBytesIndexFieldData pagedBytesIndexFieldData = createPagedBytes("field2", fieldDataCache);
    pagedBytesIndexFieldData.loadGlobal(ir);
    assertThat(fieldDataCache.cachedGlobally, equalTo(2));
    pagedBytesIndexFieldData.loadGlobal(new FieldMaskingReader("field2", ir));
    assertThat(fieldDataCache.cachedGlobally, equalTo(2));
    ir.close();
    dir.close();
}

Example 2 with FieldMaskingReader

use of org.opensearch.test.FieldMaskingReader in project OpenSearch by opensearch-project.

the class IndexShardTests method testReaderWrapperWorksWithGlobalOrdinals.

public void testReaderWrapperWorksWithGlobalOrdinals() throws IOException {
    CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = reader -> new FieldMaskingReader("foo", reader);
    Settings settings = Settings.builder().put(IndexMetadata.SETTING_VERSION_CREATED, Version.CURRENT).put(IndexMetadata.SETTING_NUMBER_OF_REPLICAS, 0).put(IndexMetadata.SETTING_NUMBER_OF_SHARDS, 1).build();
    IndexMetadata metadata = IndexMetadata.builder("test").putMapping("_doc", "{ \"properties\": { \"foo\":  { \"type\": \"text\", \"fielddata\": true }}}").settings(settings).primaryTerm(0, 1).build();
    IndexShard shard = newShard(new ShardId(metadata.getIndex(), 0), true, "n1", metadata, wrapper);
    recoverShardFromStore(shard);
    indexDoc(shard, "_doc", "0", "{\"foo\" : \"bar\"}");
    shard.refresh("created segment 1");
    indexDoc(shard, "_doc", "1", "{\"foobar\" : \"bar\"}");
    shard.refresh("created segment 2");
    // test global ordinals are evicted
    MappedFieldType foo = shard.mapperService().fieldType("foo");
    IndicesFieldDataCache indicesFieldDataCache = new IndicesFieldDataCache(shard.indexSettings.getNodeSettings(), new IndexFieldDataCache.Listener() {
    });
    IndexFieldDataService indexFieldDataService = new IndexFieldDataService(shard.indexSettings, indicesFieldDataCache, new NoneCircuitBreakerService(), shard.mapperService());
    IndexFieldData.Global ifd = indexFieldDataService.getForField(foo, "test", () -> {
        throw new UnsupportedOperationException("search lookup not available");
    });
    FieldDataStats before = shard.fieldData().stats("foo");
    assertThat(before.getMemorySizeInBytes(), equalTo(0L));
    FieldDataStats after = null;
    try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
        assertThat("we have to have more than one segment", searcher.getDirectoryReader().leaves().size(), greaterThan(1));
        ifd.loadGlobal(searcher.getDirectoryReader());
        after = shard.fieldData().stats("foo");
        assertEquals(after.getEvictions(), before.getEvictions());
        // If a field doesn't exist an empty IndexFieldData is returned and that isn't cached:
        assertThat(after.getMemorySizeInBytes(), equalTo(0L));
    }
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), after.getMemorySizeInBytes());
    shard.flush(new FlushRequest().force(true).waitIfOngoing(true));
    shard.refresh("test");
    assertEquals(shard.fieldData().stats("foo").getMemorySizeInBytes(), before.getMemorySizeInBytes());
    assertEquals(shard.fieldData().stats("foo").getEvictions(), before.getEvictions());
    closeShards(shard);
}

Example 3 with FieldMaskingReader

use of org.opensearch.test.FieldMaskingReader in project OpenSearch by opensearch-project.

the class IndexShardTests method testReaderWrapperIsUsed.

public void testReaderWrapperIsUsed() throws IOException {
    IndexShard shard = newStartedShard(true);
    indexDoc(shard, "_doc", "0", "{\"foo\" : \"bar\"}");
    indexDoc(shard, "_doc", "1", "{\"foobar\" : \"bar\"}");
    shard.refresh("test");
    try (Engine.GetResult getResult = shard.get(new Engine.Get(false, false, "1", new Term(IdFieldMapper.NAME, Uid.encodeId("1"))))) {
        assertTrue(getResult.exists());
        assertNotNull(getResult.searcher());
    }
    try (Engine.Searcher searcher = shard.acquireSearcher("test")) {
        TopDocs search = searcher.search(new TermQuery(new Term("foo", "bar")), 10);
        assertEquals(search.totalHits.value, 1);
        search = searcher.search(new TermQuery(new Term("foobar", "bar")), 10);
        assertEquals(search.totalHits.value, 1);
    }
    CheckedFunction<DirectoryReader, DirectoryReader, IOException> wrapper = reader -> new FieldMaskingReader("foo", reader);
    closeShards(shard);
    IndexShard newShard = newShard(ShardRoutingHelper.initWithSameId(shard.routingEntry(), RecoverySource.ExistingStoreRecoverySource.INSTANCE), shard.shardPath(), shard.indexSettings().getIndexMetadata(), null, wrapper, new InternalEngineFactory(), shard.getEngineConfigFactory(), () -> {
    }, RetentionLeaseSyncer.EMPTY, EMPTY_EVENT_LISTENER);
    recoverShardFromStore(newShard);
    try (Engine.Searcher searcher = newShard.acquireSearcher("test")) {
        TopDocs search = searcher.search(new TermQuery(new Term("foo", "bar")), 10);
        assertEquals(search.totalHits.value, 0);
        search = searcher.search(new TermQuery(new Term("foobar", "bar")), 10);
        assertEquals(search.totalHits.value, 1);
    }
    try (Engine.GetResult getResult = newShard.get(new Engine.Get(false, false, "1", new Term(IdFieldMapper.NAME, Uid.encodeId("1"))))) {
        assertTrue(getResult.exists());
        // make sure get uses the wrapped reader
        assertNotNull(getResult.searcher());
        assertTrue(getResult.searcher().getIndexReader() instanceof FieldMaskingReader);
    }
    closeShards(newShard);
}