Examples with Accountable org.apache.lucene.util.Accountable used on opensource projects

Example 11 with Accountable

use of org.apache.lucene.util.Accountable in project elasticsearch by elastic.

the class BitSetFilterCacheTests method testListener.

public void testListener() throws IOException {
    IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()));
    Document document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();
    final DirectoryReader writerReader = DirectoryReader.open(writer);
    final IndexReader reader = ElasticsearchDirectoryReader.wrap(writerReader, new ShardId("test", "_na_", 0));
    final AtomicLong stats = new AtomicLong();
    final AtomicInteger onCacheCalls = new AtomicInteger();
    final AtomicInteger onRemoveCalls = new AtomicInteger();
    final BitsetFilterCache cache = new BitsetFilterCache(INDEX_SETTINGS, new BitsetFilterCache.Listener() {

        @Override
        public void onCache(ShardId shardId, Accountable accountable) {
            onCacheCalls.incrementAndGet();
            stats.addAndGet(accountable.ramBytesUsed());
            if (writerReader != reader) {
                assertNotNull(shardId);
                assertEquals("test", shardId.getIndexName());
                assertEquals(0, shardId.id());
            } else {
                assertNull(shardId);
            }
        }

        @Override
        public void onRemoval(ShardId shardId, Accountable accountable) {
            onRemoveCalls.incrementAndGet();
            stats.addAndGet(-accountable.ramBytesUsed());
            if (writerReader != reader) {
                assertNotNull(shardId);
                assertEquals("test", shardId.getIndexName());
                assertEquals(0, shardId.id());
            } else {
                assertNull(shardId);
            }
        }
    });
    BitSetProducer filter = cache.getBitSetProducer(new TermQuery(new Term("field", "value")));
    assertThat(matchCount(filter, reader), equalTo(1));
    assertTrue(stats.get() > 0);
    assertEquals(1, onCacheCalls.get());
    assertEquals(0, onRemoveCalls.get());
    IOUtils.close(reader, writer);
    assertEquals(1, onRemoveCalls.get());
    assertEquals(0, stats.get());
}

Example 12 with Accountable

use of org.apache.lucene.util.Accountable in project elasticsearch by elastic.

the class BitSetFilterCacheTests method testInvalidateEntries.

public void testInvalidateEntries() throws Exception {
    IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()));
    Document document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();
    document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();
    document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();
    DirectoryReader reader = DirectoryReader.open(writer);
    reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId("test", "_na_", 0));
    IndexSearcher searcher = new IndexSearcher(reader);
    BitsetFilterCache cache = new BitsetFilterCache(INDEX_SETTINGS, new BitsetFilterCache.Listener() {

        @Override
        public void onCache(ShardId shardId, Accountable accountable) {
        }

        @Override
        public void onRemoval(ShardId shardId, Accountable accountable) {
        }
    });
    BitSetProducer filter = cache.getBitSetProducer(new TermQuery(new Term("field", "value")));
    assertThat(matchCount(filter, reader), equalTo(3));
    // now cached
    assertThat(matchCount(filter, reader), equalTo(3));
    // There are 3 segments
    assertThat(cache.getLoadedFilters().weight(), equalTo(3L));
    writer.forceMerge(1);
    reader.close();
    reader = DirectoryReader.open(writer);
    reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId("test", "_na_", 0));
    searcher = new IndexSearcher(reader);
    assertThat(matchCount(filter, reader), equalTo(3));
    // now cached
    assertThat(matchCount(filter, reader), equalTo(3));
    // Only one segment now, so the size must be 1
    assertThat(cache.getLoadedFilters().weight(), equalTo(1L));
    reader.close();
    writer.close();
    // There is no reference from readers and writer to any segment in the test index, so the size in the fbs cache must be 0
    assertThat(cache.getLoadedFilters().weight(), equalTo(0L));
}

Example 13 with Accountable

use of org.apache.lucene.util.Accountable in project elasticsearch by elastic.

the class AbstractSortTestCase method createMockShardContext.

protected QueryShardContext createMockShardContext() {
    Index index = new Index(randomAsciiOfLengthBetween(1, 10), "_na_");
    IndexSettings idxSettings = IndexSettingsModule.newIndexSettings(index, Settings.builder().put(IndexMetaData.SETTING_VERSION_CREATED, Version.CURRENT).build());
    IndicesFieldDataCache cache = new IndicesFieldDataCache(Settings.EMPTY, null);
    IndexFieldDataService ifds = new IndexFieldDataService(IndexSettingsModule.newIndexSettings("test", Settings.EMPTY), cache, null, null);
    BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(idxSettings, new BitsetFilterCache.Listener() {

        @Override
        public void onRemoval(ShardId shardId, Accountable accountable) {
        }

        @Override
        public void onCache(ShardId shardId, Accountable accountable) {
        }
    });
    long nowInMillis = randomNonNegativeLong();
    return new QueryShardContext(0, idxSettings, bitsetFilterCache, ifds, null, null, scriptService, xContentRegistry(), null, null, () -> nowInMillis) {

        @Override
        public MappedFieldType fieldMapper(String name) {
            return provideMappedFieldType(name);
        }

        @Override
        public ObjectMapper getObjectMapper(String name) {
            BuilderContext context = new BuilderContext(this.getIndexSettings().getSettings(), new ContentPath());
            return new ObjectMapper.Builder<>(name).nested(Nested.newNested(false, false)).build(context);
        }
    };
}

Example 14 with Accountable

use of org.apache.lucene.util.Accountable in project elasticsearch by elastic.

the class IndexFieldDataServiceTests method testSetCacheListenerTwice.

public void testSetCacheListenerTwice() {
    final IndexService indexService = createIndex("test");
    IndexFieldDataService shardPrivateService = indexService.fieldData();
    try {
        shardPrivateService.setListener(new IndexFieldDataCache.Listener() {

            @Override
            public void onCache(ShardId shardId, String fieldName, Accountable ramUsage) {
            }

            @Override
            public void onRemoval(ShardId shardId, String fieldName, boolean wasEvicted, long sizeInBytes) {
            }
        });
        fail("listener already set");
    } catch (IllegalStateException ex) {
    // all well
    }
}

Example 15 with Accountable

use of org.apache.lucene.util.Accountable in project lucene-solr by apache.

the class MemoryPostingsFormat method fieldsProducer.

@Override
public FieldsProducer fieldsProducer(SegmentReadState state) throws IOException {
    final String fileName = IndexFileNames.segmentFileName(state.segmentInfo.name, state.segmentSuffix, EXTENSION);
    final SortedMap<String, TermsReader> fields = new TreeMap<>();
    try (ChecksumIndexInput in = state.directory.openChecksumInput(fileName, IOContext.READONCE)) {
        Throwable priorE = null;
        try {
            CodecUtil.checkIndexHeader(in, CODEC_NAME, VERSION_START, VERSION_CURRENT, state.segmentInfo.getId(), state.segmentSuffix);
            while (true) {
                final int termCount = in.readVInt();
                if (termCount == 0) {
                    break;
                }
                final TermsReader termsReader = new TermsReader(state.fieldInfos, in, termCount);
                // System.out.println("load field=" + termsReader.field.name);
                fields.put(termsReader.field.name, termsReader);
            }
        } catch (Throwable exception) {
            priorE = exception;
        } finally {
            CodecUtil.checkFooter(in, priorE);
        }
    }
    return new FieldsProducer() {

        @Override
        public Iterator<String> iterator() {
            return Collections.unmodifiableSet(fields.keySet()).iterator();
        }

        @Override
        public Terms terms(String field) {
            return fields.get(field);
        }

        @Override
        public int size() {
            return fields.size();
        }

        @Override
        public void close() {
            // Drop ref to FST:
            for (TermsReader termsReader : fields.values()) {
                termsReader.fst = null;
            }
        }

        @Override
        public long ramBytesUsed() {
            long sizeInBytes = 0;
            for (Map.Entry<String, TermsReader> entry : fields.entrySet()) {
                sizeInBytes += (entry.getKey().length() * Character.BYTES);
                sizeInBytes += entry.getValue().ramBytesUsed();
            }
            return sizeInBytes;
        }

        @Override
        public Collection<Accountable> getChildResources() {
            return Accountables.namedAccountables("field", fields);
        }

        @Override
        public String toString() {
            return "MemoryPostings(fields=" + fields.size() + ")";
        }

        @Override
        public void checkIntegrity() throws IOException {
        }
    };
}