Examples with TermsEnum org.apache.lucene.index.TermsEnum used on opensource projects

Example 31 with TermsEnum

use of org.apache.lucene.index.TermsEnum in project lucene-solr by apache.

the class TermFreqValueSource method getValues.

@Override
public FunctionValues getValues(Map context, LeafReaderContext readerContext) throws IOException {
    Fields fields = readerContext.reader().fields();
    final Terms terms = fields.terms(indexedField);
    return new IntDocValues(this) {

        PostingsEnum docs;

        int atDoc;

        int lastDocRequested = -1;

        {
            reset();
        }

        public void reset() throws IOException {
            if (terms != null) {
                final TermsEnum termsEnum = terms.iterator();
                if (termsEnum.seekExact(indexedBytes)) {
                    docs = termsEnum.postings(null);
                } else {
                    docs = null;
                }
            } else {
                docs = null;
            }
            if (docs == null) {
                docs = new PostingsEnum() {

                    @Override
                    public int freq() {
                        return 0;
                    }

                    @Override
                    public int nextPosition() throws IOException {
                        return -1;
                    }

                    @Override
                    public int startOffset() throws IOException {
                        return -1;
                    }

                    @Override
                    public int endOffset() throws IOException {
                        return -1;
                    }

                    @Override
                    public BytesRef getPayload() throws IOException {
                        throw new UnsupportedOperationException();
                    }

                    @Override
                    public int docID() {
                        return DocIdSetIterator.NO_MORE_DOCS;
                    }

                    @Override
                    public int nextDoc() {
                        return DocIdSetIterator.NO_MORE_DOCS;
                    }

                    @Override
                    public int advance(int target) {
                        return DocIdSetIterator.NO_MORE_DOCS;
                    }

                    @Override
                    public long cost() {
                        return 0;
                    }
                };
            }
            atDoc = -1;
        }

        @Override
        public int intVal(int doc) {
            try {
                if (doc < lastDocRequested) {
                    // out-of-order access.... reset
                    reset();
                }
                lastDocRequested = doc;
                if (atDoc < doc) {
                    atDoc = docs.advance(doc);
                }
                if (atDoc > doc) {
                    // end, or because the next doc is after this doc.
                    return 0;
                }
                // a match!
                return docs.freq();
            } catch (IOException e) {
                throw new RuntimeException("caught exception in function " + description() + " : doc=" + doc, e);
            }
        }
    };
}

Example 32 with TermsEnum

use of org.apache.lucene.index.TermsEnum in project lucene-solr by apache.

the class CompletionFieldsConsumer method write.

@Override
public void write(Fields fields) throws IOException {
    delegateFieldsConsumer.write(fields);
    for (String field : fields) {
        CompletionTermWriter termWriter = new CompletionTermWriter();
        Terms terms = fields.terms(field);
        if (terms == null) {
            // this can happen from ghost fields, where the incoming Fields iterator claims a field exists but it does not
            continue;
        }
        TermsEnum termsEnum = terms.iterator();
        // write terms
        BytesRef term;
        while ((term = termsEnum.next()) != null) {
            termWriter.write(term, termsEnum);
        }
        // store lookup, if needed
        long filePointer = dictOut.getFilePointer();
        if (termWriter.finish(dictOut)) {
            seenFields.put(field, new CompletionMetaData(filePointer, termWriter.minWeight, termWriter.maxWeight, termWriter.type));
        }
    }
}

Example 33 with TermsEnum

use of org.apache.lucene.index.TermsEnum in project lucene-solr by apache.

the class SimpleFacets method getFacetTermEnumCounts.

/**
   * Returns a list of terms in the specified field along with the 
   * corresponding count of documents in the set that match that constraint.
   * This method uses the FilterCache to get the intersection count between <code>docs</code>
   * and the DocSet for each term in the filter.
   *
   * @see FacetParams#FACET_LIMIT
   * @see FacetParams#FACET_ZEROS
   * @see FacetParams#FACET_MISSING
   */
public NamedList<Integer> getFacetTermEnumCounts(SolrIndexSearcher searcher, DocSet docs, String field, int offset, int limit, int mincount, boolean missing, String sort, String prefix, Predicate<BytesRef> termFilter, boolean intersectsCheck) throws IOException {
    /* :TODO: potential optimization...
    * cache the Terms with the highest docFreq and try them first
    * don't enum if we get our max from them
    */
    // Minimum term docFreq in order to use the filterCache for that term.
    int minDfFilterCache = global.getFieldInt(field, FacetParams.FACET_ENUM_CACHE_MINDF, 0);
    // make sure we have a set that is fast for random access, if we will use it for that
    DocSet fastForRandomSet = docs;
    if (minDfFilterCache > 0 && docs instanceof SortedIntDocSet) {
        SortedIntDocSet sset = (SortedIntDocSet) docs;
        fastForRandomSet = new HashDocSet(sset.getDocs(), 0, sset.size());
    }
    IndexSchema schema = searcher.getSchema();
    FieldType ft = schema.getFieldType(field);
    assert !ft.isPointField() : "Point Fields don't support enum method";
    LeafReader r = searcher.getSlowAtomicReader();
    boolean sortByCount = sort.equals("count") || sort.equals("true");
    final int maxsize = limit >= 0 ? offset + limit : Integer.MAX_VALUE - 1;
    final BoundedTreeSet<CountPair<BytesRef, Integer>> queue = sortByCount ? new BoundedTreeSet<CountPair<BytesRef, Integer>>(maxsize) : null;
    final NamedList<Integer> res = new NamedList<>();
    // the smallest value in the top 'N' values    
    int min = mincount - 1;
    int off = offset;
    int lim = limit >= 0 ? limit : Integer.MAX_VALUE;
    BytesRef prefixTermBytes = null;
    if (prefix != null) {
        String indexedPrefix = ft.toInternal(prefix);
        prefixTermBytes = new BytesRef(indexedPrefix);
    }
    Fields fields = r.fields();
    Terms terms = fields == null ? null : fields.terms(field);
    TermsEnum termsEnum = null;
    SolrIndexSearcher.DocsEnumState deState = null;
    BytesRef term = null;
    if (terms != null) {
        termsEnum = terms.iterator();
        if (prefixTermBytes != null) {
            if (termsEnum.seekCeil(prefixTermBytes) == TermsEnum.SeekStatus.END) {
                termsEnum = null;
            } else {
                term = termsEnum.term();
            }
        } else {
            // position termsEnum on first term
            term = termsEnum.next();
        }
    }
    PostingsEnum postingsEnum = null;
    CharsRefBuilder charsRef = new CharsRefBuilder();
    if (docs.size() >= mincount) {
        while (term != null) {
            if (prefixTermBytes != null && !StringHelper.startsWith(term, prefixTermBytes))
                break;
            if (termFilter == null || termFilter.test(term)) {
                int df = termsEnum.docFreq();
                // make a large difference (for example, many terms with df=1).
                if (df > 0 && df > min) {
                    int c;
                    if (df >= minDfFilterCache) {
                        if (deState == null) {
                            deState = new SolrIndexSearcher.DocsEnumState();
                            deState.fieldName = field;
                            deState.liveDocs = r.getLiveDocs();
                            deState.termsEnum = termsEnum;
                            deState.postingsEnum = postingsEnum;
                        }
                        if (intersectsCheck) {
                            c = searcher.intersects(docs, deState) ? 1 : 0;
                        } else {
                            c = searcher.numDocs(docs, deState);
                        }
                        postingsEnum = deState.postingsEnum;
                    } else {
                        // iterate over TermDocs to calculate the intersection
                        // TODO: specialize when base docset is a bitset or hash set (skipDocs)?  or does it matter for this?
                        // TODO: do this per-segment for better efficiency (MultiDocsEnum just uses base class impl)
                        // TODO: would passing deleted docs lead to better efficiency over checking the fastForRandomSet?
                        postingsEnum = termsEnum.postings(postingsEnum, PostingsEnum.NONE);
                        c = 0;
                        if (postingsEnum instanceof MultiPostingsEnum) {
                            MultiPostingsEnum.EnumWithSlice[] subs = ((MultiPostingsEnum) postingsEnum).getSubs();
                            int numSubs = ((MultiPostingsEnum) postingsEnum).getNumSubs();
                            SEGMENTS_LOOP: for (int subindex = 0; subindex < numSubs; subindex++) {
                                MultiPostingsEnum.EnumWithSlice sub = subs[subindex];
                                if (sub.postingsEnum == null)
                                    continue;
                                int base = sub.slice.start;
                                int docid;
                                while ((docid = sub.postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
                                    if (fastForRandomSet.exists(docid + base)) {
                                        c++;
                                        if (intersectsCheck) {
                                            assert c == 1;
                                            break SEGMENTS_LOOP;
                                        }
                                    }
                                }
                            }
                        } else {
                            int docid;
                            while ((docid = postingsEnum.nextDoc()) != DocIdSetIterator.NO_MORE_DOCS) {
                                if (fastForRandomSet.exists(docid)) {
                                    c++;
                                    if (intersectsCheck) {
                                        assert c == 1;
                                        break;
                                    }
                                }
                            }
                        }
                    }
                    if (sortByCount) {
                        if (c > min) {
                            BytesRef termCopy = BytesRef.deepCopyOf(term);
                            queue.add(new CountPair<>(termCopy, c));
                            if (queue.size() >= maxsize)
                                min = queue.last().val;
                        }
                    } else {
                        if (c >= mincount && --off < 0) {
                            if (--lim < 0)
                                break;
                            ft.indexedToReadable(term, charsRef);
                            res.add(charsRef.toString(), c);
                        }
                    }
                }
            }
            term = termsEnum.next();
        }
    }
    if (sortByCount) {
        for (CountPair<BytesRef, Integer> p : queue) {
            if (--off >= 0)
                continue;
            if (--lim < 0)
                break;
            ft.indexedToReadable(p.key, charsRef);
            res.add(charsRef.toString(), p.val);
        }
    }
    if (missing) {
        res.add(null, getFieldMissingCount(searcher, docs, field));
    }
    return res;
}

Example 34 with TermsEnum

use of org.apache.lucene.index.TermsEnum in project lucene-solr by apache.

the class SolrIndexSearcher method lookupId.

/**
   * lookup the docid by the unique key field, and return the id *within* the leaf reader in the low 32 bits, and the
   * index of the leaf reader in the high 32 bits. -1 is returned if not found.
   * 
   * @lucene.internal
   */
public long lookupId(BytesRef idBytes) throws IOException {
    String field = schema.getUniqueKeyField().getName();
    for (int i = 0, c = leafContexts.size(); i < c; i++) {
        final LeafReaderContext leaf = leafContexts.get(i);
        final LeafReader reader = leaf.reader();
        final Terms terms = reader.terms(field);
        if (terms == null)
            continue;
        TermsEnum te = terms.iterator();
        if (te.seekExact(idBytes)) {
            PostingsEnum docs = te.postings(null, PostingsEnum.NONE);
            docs = BitsFilteredPostingsEnum.wrap(docs, reader.getLiveDocs());
            int id = docs.nextDoc();
            if (id == DocIdSetIterator.NO_MORE_DOCS)
                continue;
            assert docs.nextDoc() == DocIdSetIterator.NO_MORE_DOCS;
            return (((long) i) << 32) | id;
        }
    }
    return -1;
}

Example 35 with TermsEnum

use of org.apache.lucene.index.TermsEnum in project lucene-solr by apache.

the class SolrIndexSearcher method getFirstMatch.

/**
   * Returns the first document number containing the term <code>t</code> Returns -1 if no document was found. This
   * method is primarily intended for clients that want to fetch documents using a unique identifier."
   * 
   * @return the first document number containing the term
   */
public int getFirstMatch(Term t) throws IOException {
    Terms terms = leafReader.terms(t.field());
    if (terms == null)
        return -1;
    BytesRef termBytes = t.bytes();
    final TermsEnum termsEnum = terms.iterator();
    if (!termsEnum.seekExact(termBytes)) {
        return -1;
    }
    PostingsEnum docs = termsEnum.postings(null, PostingsEnum.NONE);
    docs = BitsFilteredPostingsEnum.wrap(docs, leafReader.getLiveDocs());
    int id = docs.nextDoc();
    return id == DocIdSetIterator.NO_MORE_DOCS ? -1 : id;
}