Examples with BitSetProducer org.apache.lucene.search.join.BitSetProducer used on opensource projects

Example 6 with BitSetProducer

use of org.apache.lucene.search.join.BitSetProducer in project elasticsearch by elastic.

the class BitSetFilterCacheTests method testListener.

public void testListener() throws IOException {
    IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()));
    Document document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();
    final DirectoryReader writerReader = DirectoryReader.open(writer);
    final IndexReader reader = ElasticsearchDirectoryReader.wrap(writerReader, new ShardId("test", "_na_", 0));
    final AtomicLong stats = new AtomicLong();
    final AtomicInteger onCacheCalls = new AtomicInteger();
    final AtomicInteger onRemoveCalls = new AtomicInteger();
    final BitsetFilterCache cache = new BitsetFilterCache(INDEX_SETTINGS, new BitsetFilterCache.Listener() {

        @Override
        public void onCache(ShardId shardId, Accountable accountable) {
            onCacheCalls.incrementAndGet();
            stats.addAndGet(accountable.ramBytesUsed());
            if (writerReader != reader) {
                assertNotNull(shardId);
                assertEquals("test", shardId.getIndexName());
                assertEquals(0, shardId.id());
            } else {
                assertNull(shardId);
            }
        }

        @Override
        public void onRemoval(ShardId shardId, Accountable accountable) {
            onRemoveCalls.incrementAndGet();
            stats.addAndGet(-accountable.ramBytesUsed());
            if (writerReader != reader) {
                assertNotNull(shardId);
                assertEquals("test", shardId.getIndexName());
                assertEquals(0, shardId.id());
            } else {
                assertNull(shardId);
            }
        }
    });
    BitSetProducer filter = cache.getBitSetProducer(new TermQuery(new Term("field", "value")));
    assertThat(matchCount(filter, reader), equalTo(1));
    assertTrue(stats.get() > 0);
    assertEquals(1, onCacheCalls.get());
    assertEquals(0, onRemoveCalls.get());
    IOUtils.close(reader, writer);
    assertEquals(1, onRemoveCalls.get());
    assertEquals(0, stats.get());
}

Example 7 with BitSetProducer

use of org.apache.lucene.search.join.BitSetProducer in project elasticsearch by elastic.

the class BitSetFilterCacheTests method testInvalidateEntries.

public void testInvalidateEntries() throws Exception {
    IndexWriter writer = new IndexWriter(new RAMDirectory(), new IndexWriterConfig(new StandardAnalyzer()).setMergePolicy(new LogByteSizeMergePolicy()));
    Document document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();
    document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();
    document = new Document();
    document.add(new StringField("field", "value", Field.Store.NO));
    writer.addDocument(document);
    writer.commit();
    DirectoryReader reader = DirectoryReader.open(writer);
    reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId("test", "_na_", 0));
    IndexSearcher searcher = new IndexSearcher(reader);
    BitsetFilterCache cache = new BitsetFilterCache(INDEX_SETTINGS, new BitsetFilterCache.Listener() {

        @Override
        public void onCache(ShardId shardId, Accountable accountable) {
        }

        @Override
        public void onRemoval(ShardId shardId, Accountable accountable) {
        }
    });
    BitSetProducer filter = cache.getBitSetProducer(new TermQuery(new Term("field", "value")));
    assertThat(matchCount(filter, reader), equalTo(3));
    // now cached
    assertThat(matchCount(filter, reader), equalTo(3));
    // There are 3 segments
    assertThat(cache.getLoadedFilters().weight(), equalTo(3L));
    writer.forceMerge(1);
    reader.close();
    reader = DirectoryReader.open(writer);
    reader = ElasticsearchDirectoryReader.wrap(reader, new ShardId("test", "_na_", 0));
    searcher = new IndexSearcher(reader);
    assertThat(matchCount(filter, reader), equalTo(3));
    // now cached
    assertThat(matchCount(filter, reader), equalTo(3));
    // Only one segment now, so the size must be 1
    assertThat(cache.getLoadedFilters().weight(), equalTo(1L));
    reader.close();
    writer.close();
    // There is no reference from readers and writer to any segment in the test index, so the size in the fbs cache must be 0
    assertThat(cache.getLoadedFilters().weight(), equalTo(0L));
}

Example 8 with BitSetProducer

use of org.apache.lucene.search.join.BitSetProducer in project elasticsearch by elastic.

the class NestedChildrenFilterTests method testNestedChildrenFilter.

public void testNestedChildrenFilter() throws Exception {
    int numParentDocs = scaledRandomIntBetween(0, 32);
    int maxChildDocsPerParent = scaledRandomIntBetween(8, 16);
    Directory dir = newDirectory();
    RandomIndexWriter writer = new RandomIndexWriter(random(), dir);
    for (int i = 0; i < numParentDocs; i++) {
        int numChildDocs = scaledRandomIntBetween(0, maxChildDocsPerParent);
        List<Document> docs = new ArrayList<>(numChildDocs + 1);
        for (int j = 0; j < numChildDocs; j++) {
            Document childDoc = new Document();
            childDoc.add(new StringField("type", "child", Field.Store.NO));
            docs.add(childDoc);
        }
        Document parenDoc = new Document();
        parenDoc.add(new StringField("type", "parent", Field.Store.NO));
        parenDoc.add(new LegacyIntField("num_child_docs", numChildDocs, Field.Store.YES));
        docs.add(parenDoc);
        writer.addDocuments(docs);
    }
    IndexReader reader = writer.getReader();
    writer.close();
    IndexSearcher searcher = new IndexSearcher(reader);
    FetchSubPhase.HitContext hitContext = new FetchSubPhase.HitContext();
    BitSetProducer parentFilter = new QueryBitSetProducer(new TermQuery(new Term("type", "parent")));
    Query childFilter = new TermQuery(new Term("type", "child"));
    int checkedParents = 0;
    final Weight parentsWeight = searcher.createNormalizedWeight(new TermQuery(new Term("type", "parent")), false);
    for (LeafReaderContext leaf : reader.leaves()) {
        DocIdSetIterator parents = parentsWeight.scorer(leaf).iterator();
        for (int parentDoc = parents.nextDoc(); parentDoc != DocIdSetIterator.NO_MORE_DOCS; parentDoc = parents.nextDoc()) {
            int expectedChildDocs = leaf.reader().document(parentDoc).getField("num_child_docs").numericValue().intValue();
            hitContext.reset(null, leaf, parentDoc, searcher);
            NestedChildrenQuery nestedChildrenFilter = new NestedChildrenQuery(parentFilter, childFilter, hitContext);
            TotalHitCountCollector totalHitCountCollector = new TotalHitCountCollector();
            searcher.search(new ConstantScoreQuery(nestedChildrenFilter), totalHitCountCollector);
            assertThat(totalHitCountCollector.getTotalHits(), equalTo(expectedChildDocs));
            checkedParents++;
        }
    }
    assertThat(checkedParents, equalTo(numParentDocs));
    reader.close();
    dir.close();
}

Example 9 with BitSetProducer

use of org.apache.lucene.search.join.BitSetProducer in project lucene-solr by apache.

the class ChildDocTransformer method create.

@Override
public DocTransformer create(String field, SolrParams params, SolrQueryRequest req) {
    SchemaField uniqueKeyField = req.getSchema().getUniqueKeyField();
    if (uniqueKeyField == null) {
        throw new SolrException(ErrorCode.BAD_REQUEST, " ChildDocTransformer requires the schema to have a uniqueKeyField.");
    }
    String parentFilter = params.get("parentFilter");
    if (parentFilter == null) {
        throw new SolrException(ErrorCode.BAD_REQUEST, "Parent filter should be sent as parentFilter=filterCondition");
    }
    String childFilter = params.get("childFilter");
    int limit = params.getInt("limit", 10);
    BitSetProducer parentsFilter = null;
    try {
        Query parentFilterQuery = QParser.getParser(parentFilter, req).getQuery();
        parentsFilter = new QueryBitSetProducer(new QueryWrapperFilter(parentFilterQuery));
    } catch (SyntaxError syntaxError) {
        throw new SolrException(ErrorCode.BAD_REQUEST, "Failed to create correct parent filter query");
    }
    Query childFilterQuery = null;
    if (childFilter != null) {
        try {
            childFilterQuery = QParser.getParser(childFilter, req).getQuery();
        } catch (SyntaxError syntaxError) {
            throw new SolrException(ErrorCode.BAD_REQUEST, "Failed to create correct child filter query");
        }
    }
    return new ChildDocTransformer(field, parentsFilter, uniqueKeyField, req.getSchema(), childFilterQuery, limit);
}

Example 10 with BitSetProducer

use of org.apache.lucene.search.join.BitSetProducer in project lucene-solr by apache.

the class SynonymTokenizer method testToParentBlockJoinQuery.

public void testToParentBlockJoinQuery() throws Exception {
    BitSetProducer parentFilter = new QueryBitSetProducer(new TermQuery(new Term(FIELD_NAME, "parent")));
    query = new ToParentBlockJoinQuery(new TermQuery(new Term(FIELD_NAME, "child")), parentFilter, ScoreMode.None);
    searcher = newSearcher(reader);
    hits = searcher.search(query, 100);
    int maxNumFragmentsRequired = 2;
    QueryScorer scorer = new QueryScorer(query, FIELD_NAME);
    Highlighter highlighter = new Highlighter(this, scorer);
    for (int i = 0; i < hits.totalHits; i++) {
        String text = "child document";
        TokenStream tokenStream = analyzer.tokenStream(FIELD_NAME, text);
        highlighter.setTextFragmenter(new SimpleFragmenter(40));
        highlighter.getBestFragments(tokenStream, text, maxNumFragmentsRequired, "...");
    }
    assertTrue("Failed to find correct number of highlights " + numHighlights + " found", numHighlights == 1);
}