Search in sources :

Example 36 with DirectoryReader

use of org.apache.lucene.index.DirectoryReader in project elasticsearch by elastic.

the class IndicesRequestCacheTests method testClearAllEntityIdentity.

public void testClearAllEntityIdentity() throws Exception {
    IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
    AtomicBoolean indexShard = new AtomicBoolean(true);
    ShardRequestCache requestCacheStats = new ShardRequestCache();
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
    writer.addDocument(newDoc(0, "foo"));
    DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
    TestEntity entity = new TestEntity(requestCacheStats, indexShard);
    Loader loader = new Loader(reader, 0);
    writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
    DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
    Loader secondLoader = new Loader(secondReader, 0);
    writer.updateDocument(new Term("id", "0"), newDoc(0, "baz"));
    DirectoryReader thirdReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    AtomicBoolean differentIdentity = new AtomicBoolean(true);
    TestEntity thirddEntity = new TestEntity(requestCacheStats, differentIdentity);
    Loader thirdLoader = new Loader(thirdReader, 0);
    BytesReference value1 = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
    assertEquals("foo", value1.streamInput().readString());
    BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termQuery.buildAsBytes());
    assertEquals("bar", value2.streamInput().readString());
    logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
    BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termQuery.buildAsBytes());
    assertEquals("baz", value3.streamInput().readString());
    assertEquals(3, cache.count());
    final long hitCount = requestCacheStats.stats().getHitCount();
    // clear all for the indexShard Idendity even though is't still open
    cache.clear(randomFrom(entity, secondEntity));
    cache.cleanCache();
    assertEquals(1, cache.count());
    // third has not been validated since it's a different identity
    value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termQuery.buildAsBytes());
    assertEquals(hitCount + 1, requestCacheStats.stats().getHitCount());
    assertEquals("baz", value3.streamInput().readString());
    IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
}
Also used : ShardRequestCache(org.elasticsearch.index.cache.request.ShardRequestCache) BytesReference(org.elasticsearch.common.bytes.BytesReference) DirectoryReader(org.apache.lucene.index.DirectoryReader) ElasticsearchDirectoryReader(org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader) TermQueryBuilder(org.elasticsearch.index.query.TermQueryBuilder) Term(org.apache.lucene.index.Term) ShardId(org.elasticsearch.index.shard.ShardId) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) IndexWriter(org.apache.lucene.index.IndexWriter) Directory(org.apache.lucene.store.Directory)

Example 37 with DirectoryReader

use of org.apache.lucene.index.DirectoryReader in project elasticsearch by elastic.

the class IndicesRequestCacheTests method testEviction.

public void testEviction() throws Exception {
    final ByteSizeValue size;
    {
        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
        AtomicBoolean indexShard = new AtomicBoolean(true);
        ShardRequestCache requestCacheStats = new ShardRequestCache();
        Directory dir = newDirectory();
        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
        writer.addDocument(newDoc(0, "foo"));
        DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
        Loader loader = new Loader(reader, 0);
        writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
        DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
        TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
        Loader secondLoader = new Loader(secondReader, 0);
        BytesReference value1 = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
        assertEquals("foo", value1.streamInput().readString());
        BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termQuery.buildAsBytes());
        assertEquals("bar", value2.streamInput().readString());
        size = requestCacheStats.stats().getMemorySize();
        IOUtils.close(reader, secondReader, writer, dir, cache);
    }
    IndicesRequestCache cache = new IndicesRequestCache(Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), size.getBytes() + 1 + "b").build());
    AtomicBoolean indexShard = new AtomicBoolean(true);
    ShardRequestCache requestCacheStats = new ShardRequestCache();
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
    writer.addDocument(newDoc(0, "foo"));
    DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
    TestEntity entity = new TestEntity(requestCacheStats, indexShard);
    Loader loader = new Loader(reader, 0);
    writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
    DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
    Loader secondLoader = new Loader(secondReader, 0);
    writer.updateDocument(new Term("id", "0"), newDoc(0, "baz"));
    DirectoryReader thirdReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TestEntity thirddEntity = new TestEntity(requestCacheStats, indexShard);
    Loader thirdLoader = new Loader(thirdReader, 0);
    BytesReference value1 = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
    assertEquals("foo", value1.streamInput().readString());
    BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termQuery.buildAsBytes());
    assertEquals("bar", value2.streamInput().readString());
    logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
    BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termQuery.buildAsBytes());
    assertEquals("baz", value3.streamInput().readString());
    assertEquals(2, cache.count());
    assertEquals(1, requestCacheStats.stats().getEvictions());
    IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
}
Also used : ShardRequestCache(org.elasticsearch.index.cache.request.ShardRequestCache) BytesReference(org.elasticsearch.common.bytes.BytesReference) DirectoryReader(org.apache.lucene.index.DirectoryReader) ElasticsearchDirectoryReader(org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader) ByteSizeValue(org.elasticsearch.common.unit.ByteSizeValue) TermQueryBuilder(org.elasticsearch.index.query.TermQueryBuilder) Term(org.apache.lucene.index.Term) ShardId(org.elasticsearch.index.shard.ShardId) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) IndexWriter(org.apache.lucene.index.IndexWriter) Directory(org.apache.lucene.store.Directory)

Example 38 with DirectoryReader

use of org.apache.lucene.index.DirectoryReader in project elasticsearch by elastic.

the class IndicesRequestCacheTests method testCacheDifferentReaders.

public void testCacheDifferentReaders() throws Exception {
    IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
    AtomicBoolean indexShard = new AtomicBoolean(true);
    ShardRequestCache requestCacheStats = new ShardRequestCache();
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
    writer.addDocument(newDoc(0, "foo"));
    DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
    writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
    DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    // initial cache
    TestEntity entity = new TestEntity(requestCacheStats, indexShard);
    Loader loader = new Loader(reader, 0);
    BytesReference value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
    assertEquals("foo", value.streamInput().readString());
    assertEquals(0, requestCacheStats.stats().getHitCount());
    assertEquals(1, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertFalse(loader.loadedFromCache);
    assertEquals(1, cache.count());
    assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
    final int cacheSize = requestCacheStats.stats().getMemorySize().bytesAsInt();
    assertEquals(1, cache.numRegisteredCloseListeners());
    // cache the second
    TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
    loader = new Loader(secondReader, 0);
    value = cache.getOrCompute(entity, loader, secondReader, termQuery.buildAsBytes());
    assertEquals("bar", value.streamInput().readString());
    assertEquals(0, requestCacheStats.stats().getHitCount());
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertFalse(loader.loadedFromCache);
    assertEquals(2, cache.count());
    assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > cacheSize + value.length());
    assertEquals(2, cache.numRegisteredCloseListeners());
    secondEntity = new TestEntity(requestCacheStats, indexShard);
    loader = new Loader(secondReader, 0);
    value = cache.getOrCompute(secondEntity, loader, secondReader, termQuery.buildAsBytes());
    assertEquals("bar", value.streamInput().readString());
    assertEquals(1, requestCacheStats.stats().getHitCount());
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertTrue(loader.loadedFromCache);
    assertEquals(2, cache.count());
    entity = new TestEntity(requestCacheStats, indexShard);
    loader = new Loader(reader, 0);
    value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
    assertEquals("foo", value.streamInput().readString());
    assertEquals(2, requestCacheStats.stats().getHitCount());
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertTrue(loader.loadedFromCache);
    assertEquals(2, cache.count());
    // Closing the cache doesn't change returned entities
    reader.close();
    cache.cleanCache();
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertTrue(loader.loadedFromCache);
    assertEquals(1, cache.count());
    assertEquals(cacheSize, requestCacheStats.stats().getMemorySize().bytesAsInt());
    assertEquals(1, cache.numRegisteredCloseListeners());
    // release
    if (randomBoolean()) {
        secondReader.close();
    } else {
        // closed shard but reader is still open
        indexShard.set(false);
        cache.clear(secondEntity);
    }
    cache.cleanCache();
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertTrue(loader.loadedFromCache);
    assertEquals(0, cache.count());
    assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
    IOUtils.close(secondReader, writer, dir, cache);
    assertEquals(0, cache.numRegisteredCloseListeners());
}
Also used : ShardRequestCache(org.elasticsearch.index.cache.request.ShardRequestCache) BytesReference(org.elasticsearch.common.bytes.BytesReference) DirectoryReader(org.apache.lucene.index.DirectoryReader) ElasticsearchDirectoryReader(org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader) TermQueryBuilder(org.elasticsearch.index.query.TermQueryBuilder) Term(org.apache.lucene.index.Term) ShardId(org.elasticsearch.index.shard.ShardId) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) IndexWriter(org.apache.lucene.index.IndexWriter) Directory(org.apache.lucene.store.Directory)

Example 39 with DirectoryReader

use of org.apache.lucene.index.DirectoryReader in project elasticsearch by elastic.

the class IndicesQueryCacheTests method testStatsOnEviction.

// Make sure the cache behaves correctly when a segment that is associated
// with an empty cache gets closed. In that particular case, the eviction
// callback is called with a number of evicted entries equal to 0
// see https://github.com/elastic/elasticsearch/issues/15043
public void testStatsOnEviction() throws IOException {
    Directory dir1 = newDirectory();
    IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig());
    w1.addDocument(new Document());
    DirectoryReader r1 = DirectoryReader.open(w1);
    w1.close();
    ShardId shard1 = new ShardId("index", "_na_", 0);
    r1 = ElasticsearchDirectoryReader.wrap(r1, shard1);
    IndexSearcher s1 = new IndexSearcher(r1);
    s1.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE);
    Directory dir2 = newDirectory();
    IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig());
    w2.addDocument(new Document());
    DirectoryReader r2 = DirectoryReader.open(w2);
    w2.close();
    ShardId shard2 = new ShardId("index", "_na_", 1);
    r2 = ElasticsearchDirectoryReader.wrap(r2, shard2);
    IndexSearcher s2 = new IndexSearcher(r2);
    s2.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE);
    Settings settings = Settings.builder().put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10).put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true).build();
    IndicesQueryCache cache = new IndicesQueryCache(settings);
    s1.setQueryCache(cache);
    s2.setQueryCache(cache);
    assertEquals(1, s1.count(new DummyQuery(0)));
    for (int i = 1; i <= 20; ++i) {
        assertEquals(1, s2.count(new DummyQuery(i)));
    }
    QueryCacheStats stats1 = cache.getStats(shard1);
    assertEquals(0L, stats1.getCacheSize());
    assertEquals(1L, stats1.getCacheCount());
    // this used to fail because we were evicting an empty cache on
    // the segment from r1
    IOUtils.close(r1, dir1);
    cache.onClose(shard1);
    IOUtils.close(r2, dir2);
    cache.onClose(shard2);
    // this triggers some assertions
    cache.close();
}
Also used : ShardId(org.elasticsearch.index.shard.ShardId) IndexSearcher(org.apache.lucene.search.IndexSearcher) IndicesQueryCache(org.elasticsearch.indices.IndicesQueryCache) IndexWriter(org.apache.lucene.index.IndexWriter) DirectoryReader(org.apache.lucene.index.DirectoryReader) ElasticsearchDirectoryReader(org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader) QueryCacheStats(org.elasticsearch.index.cache.query.QueryCacheStats) Document(org.apache.lucene.document.Document) Settings(org.elasticsearch.common.settings.Settings) Directory(org.apache.lucene.store.Directory)

Example 40 with DirectoryReader

use of org.apache.lucene.index.DirectoryReader in project elasticsearch by elastic.

the class IndicesQueryCacheTests method testBasics.

public void testBasics() throws IOException {
    Directory dir = newDirectory();
    IndexWriter w = new IndexWriter(dir, newIndexWriterConfig());
    w.addDocument(new Document());
    DirectoryReader r = DirectoryReader.open(w);
    w.close();
    ShardId shard = new ShardId("index", "_na_", 0);
    r = ElasticsearchDirectoryReader.wrap(r, shard);
    IndexSearcher s = new IndexSearcher(r);
    s.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE);
    Settings settings = Settings.builder().put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10).put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true).build();
    IndicesQueryCache cache = new IndicesQueryCache(settings);
    s.setQueryCache(cache);
    QueryCacheStats stats = cache.getStats(shard);
    assertEquals(0L, stats.getCacheSize());
    assertEquals(0L, stats.getCacheCount());
    assertEquals(0L, stats.getHitCount());
    assertEquals(0L, stats.getMissCount());
    assertEquals(1, s.count(new DummyQuery(0)));
    stats = cache.getStats(shard);
    assertEquals(1L, stats.getCacheSize());
    assertEquals(1L, stats.getCacheCount());
    assertEquals(0L, stats.getHitCount());
    assertEquals(1L, stats.getMissCount());
    for (int i = 1; i < 20; ++i) {
        assertEquals(1, s.count(new DummyQuery(i)));
    }
    stats = cache.getStats(shard);
    assertEquals(10L, stats.getCacheSize());
    assertEquals(20L, stats.getCacheCount());
    assertEquals(0L, stats.getHitCount());
    assertEquals(20L, stats.getMissCount());
    s.count(new DummyQuery(10));
    stats = cache.getStats(shard);
    assertEquals(10L, stats.getCacheSize());
    assertEquals(20L, stats.getCacheCount());
    assertEquals(1L, stats.getHitCount());
    assertEquals(20L, stats.getMissCount());
    IOUtils.close(r, dir);
    // got emptied, but no changes to other metrics
    stats = cache.getStats(shard);
    assertEquals(0L, stats.getCacheSize());
    assertEquals(20L, stats.getCacheCount());
    assertEquals(1L, stats.getHitCount());
    assertEquals(20L, stats.getMissCount());
    cache.onClose(shard);
    // forgot everything
    stats = cache.getStats(shard);
    assertEquals(0L, stats.getCacheSize());
    assertEquals(0L, stats.getCacheCount());
    assertEquals(0L, stats.getHitCount());
    assertEquals(0L, stats.getMissCount());
    // this triggers some assertions
    cache.close();
}
Also used : ShardId(org.elasticsearch.index.shard.ShardId) IndexSearcher(org.apache.lucene.search.IndexSearcher) IndicesQueryCache(org.elasticsearch.indices.IndicesQueryCache) IndexWriter(org.apache.lucene.index.IndexWriter) DirectoryReader(org.apache.lucene.index.DirectoryReader) ElasticsearchDirectoryReader(org.elasticsearch.common.lucene.index.ElasticsearchDirectoryReader) QueryCacheStats(org.elasticsearch.index.cache.query.QueryCacheStats) Document(org.apache.lucene.document.Document) Settings(org.elasticsearch.common.settings.Settings) Directory(org.apache.lucene.store.Directory)

Aggregations

DirectoryReader (org.apache.lucene.index.DirectoryReader)325 Document (org.apache.lucene.document.Document)217 Directory (org.apache.lucene.store.Directory)185 RandomIndexWriter (org.apache.lucene.index.RandomIndexWriter)149 IndexWriter (org.apache.lucene.index.IndexWriter)124 Term (org.apache.lucene.index.Term)123 MockAnalyzer (org.apache.lucene.analysis.MockAnalyzer)98 IndexWriterConfig (org.apache.lucene.index.IndexWriterConfig)86 IndexSearcher (org.apache.lucene.search.IndexSearcher)85 StringField (org.apache.lucene.document.StringField)59 Test (org.junit.Test)51 Analyzer (org.apache.lucene.analysis.Analyzer)50 BytesRef (org.apache.lucene.util.BytesRef)50 LeafReader (org.apache.lucene.index.LeafReader)47 ArrayList (java.util.ArrayList)44 Field (org.apache.lucene.document.Field)44 LeafReaderContext (org.apache.lucene.index.LeafReaderContext)41 TermQuery (org.apache.lucene.search.TermQuery)35 TextField (org.apache.lucene.document.TextField)32 MatchAllDocsQuery (org.apache.lucene.search.MatchAllDocsQuery)32