Examples with IndexWriter org.apache.lucene.index.IndexWriter used on opensource projects

Example 41 with IndexWriter

use of org.apache.lucene.index.IndexWriter in project elasticsearch by elastic.

the class IndicesRequestCacheTests method testInvalidate.

public void testInvalidate() throws Exception {
    ShardRequestCache requestCacheStats = new ShardRequestCache();
    IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
    writer.addDocument(newDoc(0, "foo"));
    DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
    AtomicBoolean indexShard = new AtomicBoolean(true);
    // initial cache
    TestEntity entity = new TestEntity(requestCacheStats, indexShard);
    Loader loader = new Loader(reader, 0);
    BytesReference value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
    assertEquals("foo", value.streamInput().readString());
    assertEquals(0, requestCacheStats.stats().getHitCount());
    assertEquals(1, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertFalse(loader.loadedFromCache);
    assertEquals(1, cache.count());
    // cache hit
    entity = new TestEntity(requestCacheStats, indexShard);
    loader = new Loader(reader, 0);
    value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
    assertEquals("foo", value.streamInput().readString());
    assertEquals(1, requestCacheStats.stats().getHitCount());
    assertEquals(1, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertTrue(loader.loadedFromCache);
    assertEquals(1, cache.count());
    assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
    assertEquals(1, cache.numRegisteredCloseListeners());
    // load again after invalidate
    entity = new TestEntity(requestCacheStats, indexShard);
    loader = new Loader(reader, 0);
    cache.invalidate(entity, reader, termQuery.buildAsBytes());
    value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
    assertEquals("foo", value.streamInput().readString());
    assertEquals(1, requestCacheStats.stats().getHitCount());
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertFalse(loader.loadedFromCache);
    assertEquals(1, cache.count());
    assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
    assertEquals(1, cache.numRegisteredCloseListeners());
    // release
    if (randomBoolean()) {
        reader.close();
    } else {
        // closed shard but reader is still open
        indexShard.set(false);
        cache.clear(entity);
    }
    cache.cleanCache();
    assertEquals(1, requestCacheStats.stats().getHitCount());
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertEquals(0, cache.count());
    assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
    IOUtils.close(reader, writer, dir, cache);
    assertEquals(0, cache.numRegisteredCloseListeners());
}

Example 42 with IndexWriter

use of org.apache.lucene.index.IndexWriter in project elasticsearch by elastic.

the class IndicesRequestCacheTests method testClearAllEntityIdentity.

public void testClearAllEntityIdentity() throws Exception {
    IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
    AtomicBoolean indexShard = new AtomicBoolean(true);
    ShardRequestCache requestCacheStats = new ShardRequestCache();
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
    writer.addDocument(newDoc(0, "foo"));
    DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
    TestEntity entity = new TestEntity(requestCacheStats, indexShard);
    Loader loader = new Loader(reader, 0);
    writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
    DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
    Loader secondLoader = new Loader(secondReader, 0);
    writer.updateDocument(new Term("id", "0"), newDoc(0, "baz"));
    DirectoryReader thirdReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    AtomicBoolean differentIdentity = new AtomicBoolean(true);
    TestEntity thirddEntity = new TestEntity(requestCacheStats, differentIdentity);
    Loader thirdLoader = new Loader(thirdReader, 0);
    BytesReference value1 = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
    assertEquals("foo", value1.streamInput().readString());
    BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termQuery.buildAsBytes());
    assertEquals("bar", value2.streamInput().readString());
    logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
    BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termQuery.buildAsBytes());
    assertEquals("baz", value3.streamInput().readString());
    assertEquals(3, cache.count());
    final long hitCount = requestCacheStats.stats().getHitCount();
    // clear all for the indexShard Idendity even though is't still open
    cache.clear(randomFrom(entity, secondEntity));
    cache.cleanCache();
    assertEquals(1, cache.count());
    // third has not been validated since it's a different identity
    value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termQuery.buildAsBytes());
    assertEquals(hitCount + 1, requestCacheStats.stats().getHitCount());
    assertEquals("baz", value3.streamInput().readString());
    IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
}

Example 43 with IndexWriter

use of org.apache.lucene.index.IndexWriter in project elasticsearch by elastic.

the class IndicesRequestCacheTests method testEviction.

public void testEviction() throws Exception {
    final ByteSizeValue size;
    {
        IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
        AtomicBoolean indexShard = new AtomicBoolean(true);
        ShardRequestCache requestCacheStats = new ShardRequestCache();
        Directory dir = newDirectory();
        IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
        writer.addDocument(newDoc(0, "foo"));
        DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
        TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
        TestEntity entity = new TestEntity(requestCacheStats, indexShard);
        Loader loader = new Loader(reader, 0);
        writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
        DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
        TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
        Loader secondLoader = new Loader(secondReader, 0);
        BytesReference value1 = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
        assertEquals("foo", value1.streamInput().readString());
        BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termQuery.buildAsBytes());
        assertEquals("bar", value2.streamInput().readString());
        size = requestCacheStats.stats().getMemorySize();
        IOUtils.close(reader, secondReader, writer, dir, cache);
    }
    IndicesRequestCache cache = new IndicesRequestCache(Settings.builder().put(IndicesRequestCache.INDICES_CACHE_QUERY_SIZE.getKey(), size.getBytes() + 1 + "b").build());
    AtomicBoolean indexShard = new AtomicBoolean(true);
    ShardRequestCache requestCacheStats = new ShardRequestCache();
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
    writer.addDocument(newDoc(0, "foo"));
    DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
    TestEntity entity = new TestEntity(requestCacheStats, indexShard);
    Loader loader = new Loader(reader, 0);
    writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
    DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
    Loader secondLoader = new Loader(secondReader, 0);
    writer.updateDocument(new Term("id", "0"), newDoc(0, "baz"));
    DirectoryReader thirdReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TestEntity thirddEntity = new TestEntity(requestCacheStats, indexShard);
    Loader thirdLoader = new Loader(thirdReader, 0);
    BytesReference value1 = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
    assertEquals("foo", value1.streamInput().readString());
    BytesReference value2 = cache.getOrCompute(secondEntity, secondLoader, secondReader, termQuery.buildAsBytes());
    assertEquals("bar", value2.streamInput().readString());
    logger.info("Memory size: {}", requestCacheStats.stats().getMemorySize());
    BytesReference value3 = cache.getOrCompute(thirddEntity, thirdLoader, thirdReader, termQuery.buildAsBytes());
    assertEquals("baz", value3.streamInput().readString());
    assertEquals(2, cache.count());
    assertEquals(1, requestCacheStats.stats().getEvictions());
    IOUtils.close(reader, secondReader, thirdReader, writer, dir, cache);
}

Example 44 with IndexWriter

use of org.apache.lucene.index.IndexWriter in project elasticsearch by elastic.

the class IndicesRequestCacheTests method testCacheDifferentReaders.

public void testCacheDifferentReaders() throws Exception {
    IndicesRequestCache cache = new IndicesRequestCache(Settings.EMPTY);
    AtomicBoolean indexShard = new AtomicBoolean(true);
    ShardRequestCache requestCacheStats = new ShardRequestCache();
    Directory dir = newDirectory();
    IndexWriter writer = new IndexWriter(dir, newIndexWriterConfig());
    writer.addDocument(newDoc(0, "foo"));
    DirectoryReader reader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    TermQueryBuilder termQuery = new TermQueryBuilder("id", "0");
    writer.updateDocument(new Term("id", "0"), newDoc(0, "bar"));
    DirectoryReader secondReader = ElasticsearchDirectoryReader.wrap(DirectoryReader.open(writer), new ShardId("foo", "bar", 1));
    // initial cache
    TestEntity entity = new TestEntity(requestCacheStats, indexShard);
    Loader loader = new Loader(reader, 0);
    BytesReference value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
    assertEquals("foo", value.streamInput().readString());
    assertEquals(0, requestCacheStats.stats().getHitCount());
    assertEquals(1, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertFalse(loader.loadedFromCache);
    assertEquals(1, cache.count());
    assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > value.length());
    final int cacheSize = requestCacheStats.stats().getMemorySize().bytesAsInt();
    assertEquals(1, cache.numRegisteredCloseListeners());
    // cache the second
    TestEntity secondEntity = new TestEntity(requestCacheStats, indexShard);
    loader = new Loader(secondReader, 0);
    value = cache.getOrCompute(entity, loader, secondReader, termQuery.buildAsBytes());
    assertEquals("bar", value.streamInput().readString());
    assertEquals(0, requestCacheStats.stats().getHitCount());
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertFalse(loader.loadedFromCache);
    assertEquals(2, cache.count());
    assertTrue(requestCacheStats.stats().getMemorySize().bytesAsInt() > cacheSize + value.length());
    assertEquals(2, cache.numRegisteredCloseListeners());
    secondEntity = new TestEntity(requestCacheStats, indexShard);
    loader = new Loader(secondReader, 0);
    value = cache.getOrCompute(secondEntity, loader, secondReader, termQuery.buildAsBytes());
    assertEquals("bar", value.streamInput().readString());
    assertEquals(1, requestCacheStats.stats().getHitCount());
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertTrue(loader.loadedFromCache);
    assertEquals(2, cache.count());
    entity = new TestEntity(requestCacheStats, indexShard);
    loader = new Loader(reader, 0);
    value = cache.getOrCompute(entity, loader, reader, termQuery.buildAsBytes());
    assertEquals("foo", value.streamInput().readString());
    assertEquals(2, requestCacheStats.stats().getHitCount());
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertTrue(loader.loadedFromCache);
    assertEquals(2, cache.count());
    // Closing the cache doesn't change returned entities
    reader.close();
    cache.cleanCache();
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertTrue(loader.loadedFromCache);
    assertEquals(1, cache.count());
    assertEquals(cacheSize, requestCacheStats.stats().getMemorySize().bytesAsInt());
    assertEquals(1, cache.numRegisteredCloseListeners());
    // release
    if (randomBoolean()) {
        secondReader.close();
    } else {
        // closed shard but reader is still open
        indexShard.set(false);
        cache.clear(secondEntity);
    }
    cache.cleanCache();
    assertEquals(2, requestCacheStats.stats().getMissCount());
    assertEquals(0, requestCacheStats.stats().getEvictions());
    assertTrue(loader.loadedFromCache);
    assertEquals(0, cache.count());
    assertEquals(0, requestCacheStats.stats().getMemorySize().bytesAsInt());
    IOUtils.close(secondReader, writer, dir, cache);
    assertEquals(0, cache.numRegisteredCloseListeners());
}

Example 45 with IndexWriter

use of org.apache.lucene.index.IndexWriter in project elasticsearch by elastic.

the class IndicesQueryCacheTests method testStatsOnEviction.

// Make sure the cache behaves correctly when a segment that is associated
// with an empty cache gets closed. In that particular case, the eviction
// callback is called with a number of evicted entries equal to 0
// see https://github.com/elastic/elasticsearch/issues/15043
public void testStatsOnEviction() throws IOException {
    Directory dir1 = newDirectory();
    IndexWriter w1 = new IndexWriter(dir1, newIndexWriterConfig());
    w1.addDocument(new Document());
    DirectoryReader r1 = DirectoryReader.open(w1);
    w1.close();
    ShardId shard1 = new ShardId("index", "_na_", 0);
    r1 = ElasticsearchDirectoryReader.wrap(r1, shard1);
    IndexSearcher s1 = new IndexSearcher(r1);
    s1.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE);
    Directory dir2 = newDirectory();
    IndexWriter w2 = new IndexWriter(dir2, newIndexWriterConfig());
    w2.addDocument(new Document());
    DirectoryReader r2 = DirectoryReader.open(w2);
    w2.close();
    ShardId shard2 = new ShardId("index", "_na_", 1);
    r2 = ElasticsearchDirectoryReader.wrap(r2, shard2);
    IndexSearcher s2 = new IndexSearcher(r2);
    s2.setQueryCachingPolicy(QueryCachingPolicy.ALWAYS_CACHE);
    Settings settings = Settings.builder().put(IndicesQueryCache.INDICES_CACHE_QUERY_COUNT_SETTING.getKey(), 10).put(IndicesQueryCache.INDICES_QUERIES_CACHE_ALL_SEGMENTS_SETTING.getKey(), true).build();
    IndicesQueryCache cache = new IndicesQueryCache(settings);
    s1.setQueryCache(cache);
    s2.setQueryCache(cache);
    assertEquals(1, s1.count(new DummyQuery(0)));
    for (int i = 1; i <= 20; ++i) {
        assertEquals(1, s2.count(new DummyQuery(i)));
    }
    QueryCacheStats stats1 = cache.getStats(shard1);
    assertEquals(0L, stats1.getCacheSize());
    assertEquals(1L, stats1.getCacheCount());
    // this used to fail because we were evicting an empty cache on
    // the segment from r1
    IOUtils.close(r1, dir1);
    cache.onClose(shard1);
    IOUtils.close(r2, dir2);
    cache.onClose(shard2);
    // this triggers some assertions
    cache.close();
}