Examples with IndexReaderContext org.apache.lucene.index.IndexReaderContext used on opensource projects

Example 6 with IndexReaderContext

use of org.apache.lucene.index.IndexReaderContext in project lucene-solr by apache.

the class DoubleRangeFacetCounts method count.

private void count(DoubleValuesSource valueSource, List<MatchingDocs> matchingDocs) throws IOException {
    DoubleRange[] ranges = (DoubleRange[]) this.ranges;
    LongRange[] longRanges = new LongRange[ranges.length];
    for (int i = 0; i < ranges.length; i++) {
        DoubleRange range = ranges[i];
        longRanges[i] = new LongRange(range.label, NumericUtils.doubleToSortableLong(range.min), true, NumericUtils.doubleToSortableLong(range.max), true);
    }
    LongRangeCounter counter = new LongRangeCounter(longRanges);
    int missingCount = 0;
    for (MatchingDocs hits : matchingDocs) {
        DoubleValues fv = valueSource.getValues(hits.context, null);
        totCount += hits.totalHits;
        final DocIdSetIterator fastMatchDocs;
        if (fastMatchQuery != null) {
            final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(hits.context);
            final IndexSearcher searcher = new IndexSearcher(topLevelContext);
            searcher.setQueryCache(null);
            final Weight fastMatchWeight = searcher.createNormalizedWeight(fastMatchQuery, false);
            Scorer s = fastMatchWeight.scorer(hits.context);
            if (s == null) {
                continue;
            }
            fastMatchDocs = s.iterator();
        } else {
            fastMatchDocs = null;
        }
        DocIdSetIterator docs = hits.bits.iterator();
        for (int doc = docs.nextDoc(); doc != DocIdSetIterator.NO_MORE_DOCS; ) {
            if (fastMatchDocs != null) {
                int fastMatchDoc = fastMatchDocs.docID();
                if (fastMatchDoc < doc) {
                    fastMatchDoc = fastMatchDocs.advance(doc);
                }
                if (doc != fastMatchDoc) {
                    doc = docs.advance(fastMatchDoc);
                    continue;
                }
            }
            // Skip missing docs:
            if (fv.advanceExact(doc)) {
                counter.add(NumericUtils.doubleToSortableLong(fv.doubleValue()));
            } else {
                missingCount++;
            }
            doc = docs.nextDoc();
        }
    }
    missingCount += counter.fillCounts(counts);
    totCount -= missingCount;
}

Example 7 with IndexReaderContext

use of org.apache.lucene.index.IndexReaderContext in project lucene-solr by apache.

the class TestTopDocsMerge method testSort.

void testSort(boolean useFrom) throws Exception {
    IndexReader reader = null;
    Directory dir = null;
    final int numDocs = TEST_NIGHTLY ? atLeast(1000) : atLeast(100);
    final String[] tokens = new String[] { "a", "b", "c", "d", "e" };
    if (VERBOSE) {
        System.out.println("TEST: make index");
    }
    {
        dir = newDirectory();
        final RandomIndexWriter w = new RandomIndexWriter(random(), dir);
        // w.setDoRandomForceMerge(false);
        // w.w.getConfig().setMaxBufferedDocs(atLeast(100));
        final String[] content = new String[atLeast(20)];
        for (int contentIDX = 0; contentIDX < content.length; contentIDX++) {
            final StringBuilder sb = new StringBuilder();
            final int numTokens = TestUtil.nextInt(random(), 1, 10);
            for (int tokenIDX = 0; tokenIDX < numTokens; tokenIDX++) {
                sb.append(tokens[random().nextInt(tokens.length)]).append(' ');
            }
            content[contentIDX] = sb.toString();
        }
        for (int docIDX = 0; docIDX < numDocs; docIDX++) {
            final Document doc = new Document();
            doc.add(new SortedDocValuesField("string", new BytesRef(TestUtil.randomRealisticUnicodeString(random()))));
            doc.add(newTextField("text", content[random().nextInt(content.length)], Field.Store.NO));
            doc.add(new FloatDocValuesField("float", random().nextFloat()));
            final int intValue;
            if (random().nextInt(100) == 17) {
                intValue = Integer.MIN_VALUE;
            } else if (random().nextInt(100) == 17) {
                intValue = Integer.MAX_VALUE;
            } else {
                intValue = random().nextInt();
            }
            doc.add(new NumericDocValuesField("int", intValue));
            if (VERBOSE) {
                System.out.println("  doc=" + doc);
            }
            w.addDocument(doc);
        }
        reader = w.getReader();
        w.close();
    }
    // NOTE: sometimes reader has just one segment, which is
    // important to test
    final IndexSearcher searcher = newSearcher(reader);
    final IndexReaderContext ctx = searcher.getTopReaderContext();
    final ShardSearcher[] subSearchers;
    final int[] docStarts;
    if (ctx instanceof LeafReaderContext) {
        subSearchers = new ShardSearcher[1];
        docStarts = new int[1];
        subSearchers[0] = new ShardSearcher((LeafReaderContext) ctx, ctx);
        docStarts[0] = 0;
    } else {
        final CompositeReaderContext compCTX = (CompositeReaderContext) ctx;
        final int size = compCTX.leaves().size();
        subSearchers = new ShardSearcher[size];
        docStarts = new int[size];
        int docBase = 0;
        for (int searcherIDX = 0; searcherIDX < subSearchers.length; searcherIDX++) {
            final LeafReaderContext leave = compCTX.leaves().get(searcherIDX);
            subSearchers[searcherIDX] = new ShardSearcher(leave, compCTX);
            docStarts[searcherIDX] = docBase;
            docBase += leave.reader().maxDoc();
        }
    }
    final List<SortField> sortFields = new ArrayList<>();
    sortFields.add(new SortField("string", SortField.Type.STRING, true));
    sortFields.add(new SortField("string", SortField.Type.STRING, false));
    sortFields.add(new SortField("int", SortField.Type.INT, true));
    sortFields.add(new SortField("int", SortField.Type.INT, false));
    sortFields.add(new SortField("float", SortField.Type.FLOAT, true));
    sortFields.add(new SortField("float", SortField.Type.FLOAT, false));
    sortFields.add(new SortField(null, SortField.Type.SCORE, true));
    sortFields.add(new SortField(null, SortField.Type.SCORE, false));
    sortFields.add(new SortField(null, SortField.Type.DOC, true));
    sortFields.add(new SortField(null, SortField.Type.DOC, false));
    int numIters = atLeast(300);
    for (int iter = 0; iter < numIters; iter++) {
        // TODO: custom FieldComp...
        final Query query = new TermQuery(new Term("text", tokens[random().nextInt(tokens.length)]));
        final Sort sort;
        if (random().nextInt(10) == 4) {
            // Sort by score
            sort = null;
        } else {
            final SortField[] randomSortFields = new SortField[TestUtil.nextInt(random(), 1, 3)];
            for (int sortIDX = 0; sortIDX < randomSortFields.length; sortIDX++) {
                randomSortFields[sortIDX] = sortFields.get(random().nextInt(sortFields.size()));
            }
            sort = new Sort(randomSortFields);
        }
        final int numHits = TestUtil.nextInt(random(), 1, numDocs + 5);
        if (VERBOSE) {
            System.out.println("TEST: search query=" + query + " sort=" + sort + " numHits=" + numHits);
        }
        int from = -1;
        int size = -1;
        // First search on whole index:
        final TopDocs topHits;
        if (sort == null) {
            if (useFrom) {
                TopScoreDocCollector c = TopScoreDocCollector.create(numHits);
                searcher.search(query, c);
                from = TestUtil.nextInt(random(), 0, numHits - 1);
                size = numHits - from;
                TopDocs tempTopHits = c.topDocs();
                if (from < tempTopHits.scoreDocs.length) {
                    // Can't use TopDocs#topDocs(start, howMany), since it has different behaviour when start >= hitCount
                    // than TopDocs#merge currently has
                    ScoreDoc[] newScoreDocs = new ScoreDoc[Math.min(size, tempTopHits.scoreDocs.length - from)];
                    System.arraycopy(tempTopHits.scoreDocs, from, newScoreDocs, 0, newScoreDocs.length);
                    tempTopHits.scoreDocs = newScoreDocs;
                    topHits = tempTopHits;
                } else {
                    topHits = new TopDocs(tempTopHits.totalHits, new ScoreDoc[0], tempTopHits.getMaxScore());
                }
            } else {
                topHits = searcher.search(query, numHits);
            }
        } else {
            final TopFieldCollector c = TopFieldCollector.create(sort, numHits, true, true, true);
            searcher.search(query, c);
            if (useFrom) {
                from = TestUtil.nextInt(random(), 0, numHits - 1);
                size = numHits - from;
                TopDocs tempTopHits = c.topDocs();
                if (from < tempTopHits.scoreDocs.length) {
                    // Can't use TopDocs#topDocs(start, howMany), since it has different behaviour when start >= hitCount
                    // than TopDocs#merge currently has
                    ScoreDoc[] newScoreDocs = new ScoreDoc[Math.min(size, tempTopHits.scoreDocs.length - from)];
                    System.arraycopy(tempTopHits.scoreDocs, from, newScoreDocs, 0, newScoreDocs.length);
                    tempTopHits.scoreDocs = newScoreDocs;
                    topHits = tempTopHits;
                } else {
                    topHits = new TopDocs(tempTopHits.totalHits, new ScoreDoc[0], tempTopHits.getMaxScore());
                }
            } else {
                topHits = c.topDocs(0, numHits);
            }
        }
        if (VERBOSE) {
            if (useFrom) {
                System.out.println("from=" + from + " size=" + size);
            }
            System.out.println("  top search: " + topHits.totalHits + " totalHits; hits=" + (topHits.scoreDocs == null ? "null" : topHits.scoreDocs.length + " maxScore=" + topHits.getMaxScore()));
            if (topHits.scoreDocs != null) {
                for (int hitIDX = 0; hitIDX < topHits.scoreDocs.length; hitIDX++) {
                    final ScoreDoc sd = topHits.scoreDocs[hitIDX];
                    System.out.println("    doc=" + sd.doc + " score=" + sd.score);
                }
            }
        }
        // ... then all shards:
        final Weight w = searcher.createNormalizedWeight(query, true);
        final TopDocs[] shardHits;
        if (sort == null) {
            shardHits = new TopDocs[subSearchers.length];
        } else {
            shardHits = new TopFieldDocs[subSearchers.length];
        }
        for (int shardIDX = 0; shardIDX < subSearchers.length; shardIDX++) {
            final TopDocs subHits;
            final ShardSearcher subSearcher = subSearchers[shardIDX];
            if (sort == null) {
                subHits = subSearcher.search(w, numHits);
            } else {
                final TopFieldCollector c = TopFieldCollector.create(sort, numHits, true, true, true);
                subSearcher.search(w, c);
                subHits = c.topDocs(0, numHits);
            }
            shardHits[shardIDX] = subHits;
            if (VERBOSE) {
                System.out.println("  shard=" + shardIDX + " " + subHits.totalHits + " totalHits hits=" + (subHits.scoreDocs == null ? "null" : subHits.scoreDocs.length));
                if (subHits.scoreDocs != null) {
                    for (ScoreDoc sd : subHits.scoreDocs) {
                        System.out.println("    doc=" + sd.doc + " score=" + sd.score);
                    }
                }
            }
        }
        // Merge:
        final TopDocs mergedHits;
        if (useFrom) {
            if (sort == null) {
                mergedHits = TopDocs.merge(from, size, shardHits, true);
            } else {
                mergedHits = TopDocs.merge(sort, from, size, (TopFieldDocs[]) shardHits, true);
            }
        } else {
            if (sort == null) {
                mergedHits = TopDocs.merge(numHits, shardHits);
            } else {
                mergedHits = TopDocs.merge(sort, numHits, (TopFieldDocs[]) shardHits);
            }
        }
        if (mergedHits.scoreDocs != null) {
            // Make sure the returned shards are correct:
            for (int hitIDX = 0; hitIDX < mergedHits.scoreDocs.length; hitIDX++) {
                final ScoreDoc sd = mergedHits.scoreDocs[hitIDX];
                assertEquals("doc=" + sd.doc + " wrong shard", ReaderUtil.subIndex(sd.doc, docStarts), sd.shardIndex);
            }
        }
        TestUtil.assertEquals(topHits, mergedHits);
    }
    reader.close();
    dir.close();
}

Example 8 with IndexReaderContext

use of org.apache.lucene.index.IndexReaderContext in project lucene-solr by apache.

the class TestSpans method testSpanScorerZeroSloppyFreq.

public void testSpanScorerZeroSloppyFreq() throws Exception {
    IndexReaderContext topReaderContext = searcher.getTopReaderContext();
    List<LeafReaderContext> leaves = topReaderContext.leaves();
    int subIndex = ReaderUtil.subIndex(11, leaves);
    for (int i = 0, c = leaves.size(); i < c; i++) {
        final LeafReaderContext ctx = leaves.get(i);
        final Similarity sim = new ClassicSimilarity() {

            @Override
            public float sloppyFreq(int distance) {
                return 0.0f;
            }
        };
        final Similarity oldSim = searcher.getSimilarity(true);
        Scorer spanScorer;
        try {
            searcher.setSimilarity(sim);
            SpanQuery snq = spanNearOrderedQuery(field, 1, "t1", "t2");
            spanScorer = searcher.createNormalizedWeight(snq, true).scorer(ctx);
        } finally {
            searcher.setSimilarity(oldSim);
        }
        if (i == subIndex) {
            assertTrue("first doc", spanScorer.iterator().nextDoc() != DocIdSetIterator.NO_MORE_DOCS);
            assertEquals("first doc number", spanScorer.docID() + ctx.docBase, 11);
            float score = spanScorer.score();
            assertTrue("first doc score should be zero, " + score, score == 0.0f);
        } else {
            assertTrue("no second doc", spanScorer == null || spanScorer.iterator().nextDoc() == DocIdSetIterator.NO_MORE_DOCS);
        }
    }
}

Example 9 with IndexReaderContext

use of org.apache.lucene.index.IndexReaderContext in project lucene-solr by apache.

the class QueryBitSetProducer method getBitSet.

@Override
public BitSet getBitSet(LeafReaderContext context) throws IOException {
    final LeafReader reader = context.reader();
    final IndexReader.CacheHelper cacheHelper = reader.getCoreCacheHelper();
    DocIdSet docIdSet = null;
    if (cacheHelper != null) {
        docIdSet = cache.get(cacheHelper.getKey());
    }
    if (docIdSet == null) {
        final IndexReaderContext topLevelContext = ReaderUtil.getTopLevelContext(context);
        final IndexSearcher searcher = new IndexSearcher(topLevelContext);
        searcher.setQueryCache(null);
        final Weight weight = searcher.createNormalizedWeight(query, false);
        final Scorer s = weight.scorer(context);
        if (s == null) {
            docIdSet = DocIdSet.EMPTY;
        } else {
            docIdSet = new BitDocIdSet(BitSet.of(s.iterator(), context.reader().maxDoc()));
        }
        if (cacheHelper != null) {
            cache.put(cacheHelper.getKey(), docIdSet);
        }
    }
    return docIdSet == DocIdSet.EMPTY ? null : ((BitDocIdSet) docIdSet).bits();
}

Example 10 with IndexReaderContext

use of org.apache.lucene.index.IndexReaderContext in project elasticsearch by elastic.

the class DfsPhase method execute.

@Override
public void execute(SearchContext context) {
    final ObjectHashSet<Term> termsSet = new ObjectHashSet<>();
    try {
        context.searcher().createNormalizedWeight(context.query(), true).extractTerms(new DelegateSet(termsSet));
        for (RescoreSearchContext rescoreContext : context.rescore()) {
            rescoreContext.rescorer().extractTerms(context, rescoreContext, new DelegateSet(termsSet));
        }
        Term[] terms = termsSet.toArray(Term.class);
        TermStatistics[] termStatistics = new TermStatistics[terms.length];
        IndexReaderContext indexReaderContext = context.searcher().getTopReaderContext();
        for (int i = 0; i < terms.length; i++) {
            if (context.isCancelled()) {
                throw new TaskCancelledException("cancelled");
            }
            // LUCENE 4 UPGRADE: cache TermContext?
            TermContext termContext = TermContext.build(indexReaderContext, terms[i]);
            termStatistics[i] = context.searcher().termStatistics(terms[i], termContext);
        }
        ObjectObjectHashMap<String, CollectionStatistics> fieldStatistics = HppcMaps.newNoNullKeysMap();
        for (Term term : terms) {
            assert term.field() != null : "field is null";
            if (!fieldStatistics.containsKey(term.field())) {
                final CollectionStatistics collectionStatistics = context.searcher().collectionStatistics(term.field());
                fieldStatistics.put(term.field(), collectionStatistics);
                if (context.isCancelled()) {
                    throw new TaskCancelledException("cancelled");
                }
            }
        }
        context.dfsResult().termsStatistics(terms, termStatistics).fieldStatistics(fieldStatistics).maxDoc(context.searcher().getIndexReader().maxDoc());
    } catch (Exception e) {
        throw new DfsPhaseExecutionException(context, "Exception during dfs phase", e);
    } finally {
        // don't hold on to terms
        termsSet.clear();
    }
}