Examples with DocIterator org.apache.solr.search.DocIterator used on opensource projects

Example 21 with DocIterator

use of org.apache.solr.search.DocIterator in project lucene-solr by apache.

the class QueryComponent method doFieldSortValues.

protected void doFieldSortValues(ResponseBuilder rb, SolrIndexSearcher searcher) throws IOException {
    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;
    // The query cache doesn't currently store sort field values, and SolrIndexSearcher doesn't
    // currently have an option to return sort field values.  Because of this, we
    // take the documents given and re-derive the sort values.
    //
    // TODO: See SOLR-5595
    boolean fsv = req.getParams().getBool(ResponseBuilder.FIELD_SORT_VALUES, false);
    if (fsv) {
        // order is important for the sort fields
        NamedList<Object[]> sortVals = new NamedList<>();
        IndexReaderContext topReaderContext = searcher.getTopReaderContext();
        List<LeafReaderContext> leaves = topReaderContext.leaves();
        LeafReaderContext currentLeaf = null;
        if (leaves.size() == 1) {
            // if there is a single segment, use that subReader and avoid looking up each time
            currentLeaf = leaves.get(0);
            leaves = null;
        }
        DocList docList = rb.getResults().docList;
        // sort ids from lowest to highest so we can access them in order
        int nDocs = docList.size();
        final long[] sortedIds = new long[nDocs];
        // doc scores, parallel to sortedIds
        final float[] scores = new float[nDocs];
        DocList docs = rb.getResults().docList;
        DocIterator it = docs.iterator();
        for (int i = 0; i < nDocs; i++) {
            sortedIds[i] = (((long) it.nextDoc()) << 32) | i;
            scores[i] = docs.hasScores() ? it.score() : Float.NaN;
        }
        // sort ids and scores together
        new InPlaceMergeSorter() {

            @Override
            protected void swap(int i, int j) {
                long tmpId = sortedIds[i];
                float tmpScore = scores[i];
                sortedIds[i] = sortedIds[j];
                scores[i] = scores[j];
                sortedIds[j] = tmpId;
                scores[j] = tmpScore;
            }

            @Override
            protected int compare(int i, int j) {
                return Long.compare(sortedIds[i], sortedIds[j]);
            }
        }.sort(0, sortedIds.length);
        SortSpec sortSpec = rb.getSortSpec();
        Sort sort = searcher.weightSort(sortSpec.getSort());
        SortField[] sortFields = sort == null ? new SortField[] { SortField.FIELD_SCORE } : sort.getSort();
        List<SchemaField> schemaFields = sortSpec.getSchemaFields();
        for (int fld = 0; fld < schemaFields.size(); fld++) {
            SchemaField schemaField = schemaFields.get(fld);
            FieldType ft = null == schemaField ? null : schemaField.getType();
            SortField sortField = sortFields[fld];
            SortField.Type type = sortField.getType();
            // :TODO: would be simpler to always serialize every position of SortField[]
            if (type == SortField.Type.SCORE || type == SortField.Type.DOC)
                continue;
            FieldComparator<?> comparator = sortField.getComparator(1, 0);
            LeafFieldComparator leafComparator = null;
            Object[] vals = new Object[nDocs];
            int lastIdx = -1;
            int idx = 0;
            for (int i = 0; i < sortedIds.length; ++i) {
                long idAndPos = sortedIds[i];
                float score = scores[i];
                int doc = (int) (idAndPos >>> 32);
                int position = (int) idAndPos;
                if (leaves != null) {
                    idx = ReaderUtil.subIndex(doc, leaves);
                    currentLeaf = leaves.get(idx);
                    if (idx != lastIdx) {
                        // we switched segments.  invalidate leafComparator.
                        lastIdx = idx;
                        leafComparator = null;
                    }
                }
                if (leafComparator == null) {
                    leafComparator = comparator.getLeafComparator(currentLeaf);
                }
                // adjust for what segment this is in
                doc -= currentLeaf.docBase;
                leafComparator.setScorer(new FakeScorer(doc, score));
                leafComparator.copy(0, doc);
                Object val = comparator.value(0);
                if (null != ft)
                    val = ft.marshalSortValue(val);
                vals[position] = val;
            }
            sortVals.add(sortField.getField(), vals);
        }
        rsp.add("sort_values", sortVals);
    }
}

Example 22 with DocIterator

use of org.apache.solr.search.DocIterator in project lucene-solr by apache.

the class ClusteringComponent method docListToSolrDocumentList.

/**
   * Convert a DocList to a SolrDocumentList
   *
   * The optional param "ids" is populated with the lucene document id
   * for each SolrDocument.
   *
   * @param docs The {@link org.apache.solr.search.DocList} to convert
   * @param searcher The {@link org.apache.solr.search.SolrIndexSearcher} to use to load the docs from the Lucene index
   * @param fields The names of the Fields to load
   * @param ids A map to store the ids of the docs
   * @return The new {@link SolrDocumentList} containing all the loaded docs
   * @throws IOException if there was a problem loading the docs
   * @since solr 1.4
   */
public static SolrDocumentList docListToSolrDocumentList(DocList docs, SolrIndexSearcher searcher, Set<String> fields, Map<SolrDocument, Integer> ids) throws IOException {
    IndexSchema schema = searcher.getSchema();
    SolrDocumentList list = new SolrDocumentList();
    list.setNumFound(docs.matches());
    list.setMaxScore(docs.maxScore());
    list.setStart(docs.offset());
    DocIterator dit = docs.iterator();
    while (dit.hasNext()) {
        int docid = dit.nextDoc();
        Document luceneDoc = searcher.doc(docid, fields);
        SolrDocument doc = new SolrDocument();
        for (IndexableField field : luceneDoc) {
            if (null == fields || fields.contains(field.name())) {
                SchemaField sf = schema.getField(field.name());
                doc.addField(field.name(), sf.getType().toObject(field));
            }
        }
        if (docs.hasScores() && (null == fields || fields.contains("score"))) {
            doc.addField("score", dit.score());
        }
        list.add(doc);
        if (ids != null) {
            ids.put(doc, new Integer(docid));
        }
    }
    return list;
}

Example 23 with DocIterator

use of org.apache.solr.search.DocIterator in project lucene-solr by apache.

the class QuerySenderListener method newSearcher.

@Override
public void newSearcher(SolrIndexSearcher newSearcher, SolrIndexSearcher currentSearcher) {
    final SolrIndexSearcher searcher = newSearcher;
    log.info("QuerySenderListener sending requests to " + newSearcher);
    List<NamedList> allLists = (List<NamedList>) getArgs().get("queries");
    if (allLists == null)
        return;
    boolean createNewReqInfo = SolrRequestInfo.getRequestInfo() == null;
    for (NamedList nlst : allLists) {
        SolrQueryRequest req = null;
        try {
            // bind the request to a particular searcher (the newSearcher)
            NamedList params = addEventParms(currentSearcher, nlst);
            // for this, we default to distrib = false
            if (params.get(DISTRIB) == null) {
                params.add(DISTRIB, false);
            }
            req = new LocalSolrQueryRequest(getCore(), params) {

                @Override
                public SolrIndexSearcher getSearcher() {
                    return searcher;
                }

                @Override
                public void close() {
                }
            };
            SolrQueryResponse rsp = new SolrQueryResponse();
            if (createNewReqInfo) {
                // SolrRequerstInfo for this thread could have been transferred from the parent
                // thread.
                SolrRequestInfo.setRequestInfo(new SolrRequestInfo(req, rsp));
            }
            getCore().execute(getCore().getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp);
            // Retrieve the Document instances (not just the ids) to warm
            // the OS disk cache, and any Solr document cache.  Only the top
            // level values in the NamedList are checked for DocLists.
            NamedList values = rsp.getValues();
            for (int i = 0; i < values.size(); i++) {
                Object o = values.getVal(i);
                if (o instanceof ResultContext) {
                    o = ((ResultContext) o).getDocList();
                }
                if (o instanceof DocList) {
                    DocList docs = (DocList) o;
                    for (DocIterator iter = docs.iterator(); iter.hasNext(); ) {
                        newSearcher.doc(iter.nextDoc());
                    }
                }
            }
        } catch (Exception e) {
        // do nothing... we want to continue with the other requests.
        // the failure should have already been logged.
        } finally {
            if (req != null)
                req.close();
            if (createNewReqInfo)
                SolrRequestInfo.clearRequestInfo();
        }
    }
    log.info("QuerySenderListener done.");
}

Example 24 with DocIterator

use of org.apache.solr.search.DocIterator in project lucene-solr by apache.

the class IntervalFacets method getCountMultiValuedNumeric.

private void getCountMultiValuedNumeric() throws IOException {
    final FieldType ft = schemaField.getType();
    final String fieldName = schemaField.getName();
    if (ft.getNumberType() == null) {
        throw new IllegalStateException();
    }
    final List<LeafReaderContext> leaves = searcher.getIndexReader().leaves();
    final Iterator<LeafReaderContext> ctxIt = leaves.iterator();
    LeafReaderContext ctx = null;
    SortedNumericDocValues longs = null;
    for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
        final int doc = docsIt.nextDoc();
        if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) {
            do {
                ctx = ctxIt.next();
            } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc());
            assert doc >= ctx.docBase;
            longs = DocValues.getSortedNumeric(ctx.reader(), fieldName);
        }
        int valuesDocID = longs.docID();
        if (valuesDocID < doc - ctx.docBase) {
            valuesDocID = longs.advance(doc - ctx.docBase);
        }
        if (valuesDocID == doc - ctx.docBase) {
            accumIntervalWithMultipleValues(longs);
        }
    }
}

Example 25 with DocIterator

use of org.apache.solr.search.DocIterator in project lucene-solr by apache.

the class NumericFacets method getCountsMultiValued.

private static NamedList<Integer> getCountsMultiValued(SolrIndexSearcher searcher, DocSet docs, String fieldName, int offset, int limit, int mincount, boolean missing, String sort) throws IOException {
    // If facet.mincount=0 with PointFields the only option is to get the values from DocValues
    // not currently supported. See SOLR-10033
    mincount = Math.max(mincount, 1);
    final SchemaField sf = searcher.getSchema().getField(fieldName);
    final FieldType ft = sf.getType();
    assert sf.multiValued();
    final List<LeafReaderContext> leaves = searcher.getIndexReader().leaves();
    // 1. accumulate
    final HashTable hashTable = new HashTable(false);
    final Iterator<LeafReaderContext> ctxIt = leaves.iterator();
    LeafReaderContext ctx = null;
    SortedNumericDocValues longs = null;
    int missingCount = 0;
    for (DocIterator docsIt = docs.iterator(); docsIt.hasNext(); ) {
        final int doc = docsIt.nextDoc();
        if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) {
            do {
                ctx = ctxIt.next();
            } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc());
            assert doc >= ctx.docBase;
            longs = DocValues.getSortedNumeric(ctx.reader(), fieldName);
        }
        int valuesDocID = longs.docID();
        if (valuesDocID < doc - ctx.docBase) {
            valuesDocID = longs.advance(doc - ctx.docBase);
        }
        if (valuesDocID == doc - ctx.docBase) {
            // This document must have at least one value
            long l = longs.nextValue();
            hashTable.add(l, 1);
            for (int i = 1; i < longs.docValueCount(); i++) {
                long lnew = longs.nextValue();
                if (lnew > l) {
                    // Skip the value if it's equal to the last one, we don't want to double-count it
                    hashTable.add(lnew, 1);
                }
                l = lnew;
            }
        } else {
            ++missingCount;
        }
    }
    // 2. select top-k facet values
    final int pqSize = limit < 0 ? hashTable.size : Math.min(offset + limit, hashTable.size);
    final PriorityQueue<Entry> pq;
    if (FacetParams.FACET_SORT_COUNT.equals(sort) || FacetParams.FACET_SORT_COUNT_LEGACY.equals(sort)) {
        pq = new PriorityQueue<Entry>(pqSize) {

            @Override
            protected boolean lessThan(Entry a, Entry b) {
                if (a.count < b.count || (a.count == b.count && a.bits > b.bits)) {
                    return true;
                } else {
                    return false;
                }
            }
        };
    } else {
        // sort=index
        pq = new PriorityQueue<Entry>(pqSize) {

            @Override
            protected boolean lessThan(Entry a, Entry b) {
                return a.bits > b.bits;
            }
        };
    }
    Entry e = null;
    for (int i = 0; i < hashTable.bits.length; ++i) {
        if (hashTable.counts[i] >= mincount) {
            if (e == null) {
                e = new Entry();
            }
            e.bits = hashTable.bits[i];
            e.count = hashTable.counts[i];
            e = pq.insertWithOverflow(e);
        }
    }
    // 4. build the NamedList
    final NamedList<Integer> result = new NamedList<>(Math.max(pq.size() - offset + 1, 1));
    final Deque<Entry> counts = new ArrayDeque<>(pq.size() - offset);
    while (pq.size() > offset) {
        counts.addFirst(pq.pop());
    }
    for (Entry entry : counts) {
        // TODO: convert to correct value
        result.add(bitsToStringValue(ft, entry.bits), entry.count);
    }
    if (missing) {
        result.add(null, missingCount);
    }
    return result;
}