Search in sources :

Example 11 with DocIterator

use of org.apache.solr.search.DocIterator in project lucene-solr by apache.

the class ExpandComponent method process.

@SuppressWarnings("unchecked")
@Override
public void process(ResponseBuilder rb) throws IOException {
    if (!rb.doExpand) {
        return;
    }
    SolrQueryRequest req = rb.req;
    SolrParams params = req.getParams();
    String field = params.get(ExpandParams.EXPAND_FIELD);
    String hint = null;
    if (field == null) {
        List<Query> filters = rb.getFilters();
        if (filters != null) {
            for (Query q : filters) {
                if (q instanceof CollapsingQParserPlugin.CollapsingPostFilter) {
                    CollapsingQParserPlugin.CollapsingPostFilter cp = (CollapsingQParserPlugin.CollapsingPostFilter) q;
                    field = cp.getField();
                    hint = cp.hint;
                }
            }
        }
    }
    if (field == null) {
        throw new IOException("Expand field is null.");
    }
    String sortParam = params.get(ExpandParams.EXPAND_SORT);
    String[] fqs = params.getParams(ExpandParams.EXPAND_FQ);
    String qs = params.get(ExpandParams.EXPAND_Q);
    int limit = params.getInt(ExpandParams.EXPAND_ROWS, 5);
    Sort sort = null;
    if (sortParam != null) {
        sort = SortSpecParsing.parseSortSpec(sortParam, rb.req).getSort();
    }
    Query query;
    if (qs == null) {
        query = rb.getQuery();
    } else {
        try {
            QParser parser = QParser.getParser(qs, req);
            query = parser.getQuery();
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
    List<Query> newFilters = new ArrayList<>();
    if (fqs == null) {
        List<Query> filters = rb.getFilters();
        if (filters != null) {
            for (Query q : filters) {
                if (!(q instanceof CollapsingQParserPlugin.CollapsingPostFilter)) {
                    newFilters.add(q);
                }
            }
        }
    } else {
        try {
            for (String fq : fqs) {
                if (fq != null && fq.trim().length() != 0 && !fq.equals("*:*")) {
                    QParser fqp = QParser.getParser(fq, req);
                    newFilters.add(fqp.getQuery());
                }
            }
        } catch (Exception e) {
            throw new IOException(e);
        }
    }
    SolrIndexSearcher searcher = req.getSearcher();
    LeafReader reader = searcher.getSlowAtomicReader();
    SchemaField schemaField = searcher.getSchema().getField(field);
    FieldType fieldType = schemaField.getType();
    SortedDocValues values = null;
    long nullValue = 0L;
    if (fieldType instanceof StrField) {
        //Get The Top Level SortedDocValues
        if (CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
            Map<String, UninvertingReader.Type> mapping = new HashMap();
            mapping.put(field, UninvertingReader.Type.SORTED);
            UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), field), mapping);
            values = uninvertingReader.getSortedDocValues(field);
        } else {
            values = DocValues.getSorted(reader, field);
        }
    } else {
        //Get the nullValue for the numeric collapse field
        String defaultValue = searcher.getSchema().getField(field).getDefaultValue();
        final NumberType numType = fieldType.getNumberType();
        // we don't need to handle invalid 64-bit field types here.
        if (defaultValue != null) {
            if (numType == NumberType.INTEGER) {
                nullValue = Long.parseLong(defaultValue);
            } else if (numType == NumberType.FLOAT) {
                nullValue = Float.floatToIntBits(Float.parseFloat(defaultValue));
            }
        } else if (NumberType.FLOAT.equals(numType)) {
            // Integer case already handled by nullValue defaulting to 0
            nullValue = Float.floatToIntBits(0.0f);
        }
    }
    FixedBitSet groupBits = null;
    LongHashSet groupSet = null;
    DocList docList = rb.getResults().docList;
    IntHashSet collapsedSet = new IntHashSet(docList.size() * 2);
    //Gather the groups for the current page of documents
    DocIterator idit = docList.iterator();
    int[] globalDocs = new int[docList.size()];
    int docsIndex = -1;
    while (idit.hasNext()) {
        globalDocs[++docsIndex] = idit.nextDoc();
    }
    Arrays.sort(globalDocs);
    Query groupQuery = null;
    /*
    * This code gathers the group information for the current page.
    */
    List<LeafReaderContext> contexts = searcher.getTopReaderContext().leaves();
    if (contexts.size() == 0) {
        //When no context is available we can skip the expanding
        return;
    }
    int currentContext = 0;
    int currentDocBase = contexts.get(currentContext).docBase;
    int nextDocBase = (currentContext + 1) < contexts.size() ? contexts.get(currentContext + 1).docBase : Integer.MAX_VALUE;
    IntObjectHashMap<BytesRef> ordBytes = null;
    if (values != null) {
        groupBits = new FixedBitSet(values.getValueCount());
        MultiDocValues.OrdinalMap ordinalMap = null;
        SortedDocValues[] sortedDocValues = null;
        LongValues segmentOrdinalMap = null;
        SortedDocValues currentValues = null;
        if (values instanceof MultiDocValues.MultiSortedDocValues) {
            ordinalMap = ((MultiDocValues.MultiSortedDocValues) values).mapping;
            sortedDocValues = ((MultiDocValues.MultiSortedDocValues) values).values;
            currentValues = sortedDocValues[currentContext];
            segmentOrdinalMap = ordinalMap.getGlobalOrds(currentContext);
        }
        int count = 0;
        ordBytes = new IntObjectHashMap<>();
        for (int i = 0; i < globalDocs.length; i++) {
            int globalDoc = globalDocs[i];
            while (globalDoc >= nextDocBase) {
                currentContext++;
                currentDocBase = contexts.get(currentContext).docBase;
                nextDocBase = (currentContext + 1) < contexts.size() ? contexts.get(currentContext + 1).docBase : Integer.MAX_VALUE;
                if (ordinalMap != null) {
                    currentValues = sortedDocValues[currentContext];
                    segmentOrdinalMap = ordinalMap.getGlobalOrds(currentContext);
                }
            }
            int contextDoc = globalDoc - currentDocBase;
            if (ordinalMap != null) {
                if (contextDoc > currentValues.docID()) {
                    currentValues.advance(contextDoc);
                }
                if (contextDoc == currentValues.docID()) {
                    int ord = currentValues.ordValue();
                    ++count;
                    BytesRef ref = currentValues.lookupOrd(ord);
                    ord = (int) segmentOrdinalMap.get(ord);
                    ordBytes.put(ord, BytesRef.deepCopyOf(ref));
                    groupBits.set(ord);
                    collapsedSet.add(globalDoc);
                }
            } else {
                if (globalDoc > values.docID()) {
                    values.advance(globalDoc);
                }
                if (globalDoc == values.docID()) {
                    int ord = values.ordValue();
                    ++count;
                    BytesRef ref = values.lookupOrd(ord);
                    ordBytes.put(ord, BytesRef.deepCopyOf(ref));
                    groupBits.set(ord);
                    collapsedSet.add(globalDoc);
                }
            }
        }
        if (count > 0 && count < 200) {
            try {
                groupQuery = getGroupQuery(field, count, ordBytes);
            } catch (Exception e) {
                throw new IOException(e);
            }
        }
    } else {
        groupSet = new LongHashSet(docList.size());
        NumericDocValues collapseValues = contexts.get(currentContext).reader().getNumericDocValues(field);
        int count = 0;
        for (int i = 0; i < globalDocs.length; i++) {
            int globalDoc = globalDocs[i];
            while (globalDoc >= nextDocBase) {
                currentContext++;
                currentDocBase = contexts.get(currentContext).docBase;
                nextDocBase = currentContext + 1 < contexts.size() ? contexts.get(currentContext + 1).docBase : Integer.MAX_VALUE;
                collapseValues = contexts.get(currentContext).reader().getNumericDocValues(field);
            }
            int contextDoc = globalDoc - currentDocBase;
            int valueDocID = collapseValues.docID();
            if (valueDocID < contextDoc) {
                valueDocID = collapseValues.advance(contextDoc);
            }
            long value;
            if (valueDocID == contextDoc) {
                value = collapseValues.longValue();
            } else {
                value = 0;
            }
            if (value != nullValue) {
                ++count;
                groupSet.add(value);
                collapsedSet.add(globalDoc);
            }
        }
        if (count > 0 && count < 200) {
            if (fieldType.isPointField()) {
                groupQuery = getPointGroupQuery(schemaField, count, groupSet);
            } else {
                groupQuery = getGroupQuery(field, fieldType, count, groupSet);
            }
        }
    }
    Collector collector;
    if (sort != null)
        sort = sort.rewrite(searcher);
    Collector groupExpandCollector = null;
    if (values != null) {
        //Get The Top Level SortedDocValues again so we can re-iterate:
        if (CollapsingQParserPlugin.HINT_TOP_FC.equals(hint)) {
            Map<String, UninvertingReader.Type> mapping = new HashMap();
            mapping.put(field, UninvertingReader.Type.SORTED);
            UninvertingReader uninvertingReader = new UninvertingReader(new ReaderWrapper(searcher.getSlowAtomicReader(), field), mapping);
            values = uninvertingReader.getSortedDocValues(field);
        } else {
            values = DocValues.getSorted(reader, field);
        }
        groupExpandCollector = new GroupExpandCollector(values, groupBits, collapsedSet, limit, sort);
    } else {
        groupExpandCollector = new NumericGroupExpandCollector(field, nullValue, groupSet, collapsedSet, limit, sort);
    }
    if (groupQuery != null) {
        //Limits the results to documents that are in the same group as the documents in the page.
        newFilters.add(groupQuery);
    }
    SolrIndexSearcher.ProcessedFilter pfilter = searcher.getProcessedFilter(null, newFilters);
    if (pfilter.postFilter != null) {
        pfilter.postFilter.setLastDelegate(groupExpandCollector);
        collector = pfilter.postFilter;
    } else {
        collector = groupExpandCollector;
    }
    if (pfilter.filter == null) {
        searcher.search(query, collector);
    } else {
        Query q = new BooleanQuery.Builder().add(query, Occur.MUST).add(pfilter.filter, Occur.FILTER).build();
        searcher.search(q, collector);
    }
    LongObjectMap<Collector> groups = ((GroupCollector) groupExpandCollector).getGroups();
    NamedList outMap = new SimpleOrderedMap();
    CharsRefBuilder charsRef = new CharsRefBuilder();
    for (LongObjectCursor<Collector> cursor : groups) {
        long groupValue = cursor.key;
        TopDocsCollector<?> topDocsCollector = TopDocsCollector.class.cast(cursor.value);
        TopDocs topDocs = topDocsCollector.topDocs();
        ScoreDoc[] scoreDocs = topDocs.scoreDocs;
        if (scoreDocs.length > 0) {
            int[] docs = new int[scoreDocs.length];
            float[] scores = new float[scoreDocs.length];
            for (int i = 0; i < docs.length; i++) {
                ScoreDoc scoreDoc = scoreDocs[i];
                docs[i] = scoreDoc.doc;
                scores[i] = scoreDoc.score;
            }
            DocSlice slice = new DocSlice(0, docs.length, docs, scores, topDocs.totalHits, topDocs.getMaxScore());
            if (fieldType instanceof StrField) {
                final BytesRef bytesRef = ordBytes.get((int) groupValue);
                fieldType.indexedToReadable(bytesRef, charsRef);
                String group = charsRef.toString();
                outMap.add(group, slice);
            } else {
                outMap.add(numericToString(fieldType, groupValue), slice);
            }
        }
    }
    rb.rsp.add("expanded", outMap);
}
Also used : StrField(org.apache.solr.schema.StrField) BooleanQuery(org.apache.lucene.search.BooleanQuery) Query(org.apache.lucene.search.Query) TermInSetQuery(org.apache.lucene.search.TermInSetQuery) SolrConstantScoreQuery(org.apache.solr.search.SolrConstantScoreQuery) BooleanQuery(org.apache.lucene.search.BooleanQuery) HashMap(java.util.HashMap) LongObjectHashMap(com.carrotsearch.hppc.LongObjectHashMap) IntObjectHashMap(com.carrotsearch.hppc.IntObjectHashMap) ArrayList(java.util.ArrayList) IntHashSet(com.carrotsearch.hppc.IntHashSet) MultiDocValues(org.apache.lucene.index.MultiDocValues) DocSlice(org.apache.solr.search.DocSlice) ScoreDoc(org.apache.lucene.search.ScoreDoc) FixedBitSet(org.apache.lucene.util.FixedBitSet) Sort(org.apache.lucene.search.Sort) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) CharsRefBuilder(org.apache.lucene.util.CharsRefBuilder) SortedDocValues(org.apache.lucene.index.SortedDocValues) LongHashSet(com.carrotsearch.hppc.LongHashSet) SolrQueryRequest(org.apache.solr.request.SolrQueryRequest) NumberType(org.apache.solr.schema.NumberType) QParser(org.apache.solr.search.QParser) SolrParams(org.apache.solr.common.params.SolrParams) NumericDocValues(org.apache.lucene.index.NumericDocValues) DocIterator(org.apache.solr.search.DocIterator) SimpleOrderedMap(org.apache.solr.common.util.SimpleOrderedMap) UninvertingReader(org.apache.solr.uninverting.UninvertingReader) TopDocs(org.apache.lucene.search.TopDocs) TopFieldCollector(org.apache.lucene.search.TopFieldCollector) LeafCollector(org.apache.lucene.search.LeafCollector) Collector(org.apache.lucene.search.Collector) TopScoreDocCollector(org.apache.lucene.search.TopScoreDocCollector) TopDocsCollector(org.apache.lucene.search.TopDocsCollector) BytesRef(org.apache.lucene.util.BytesRef) LeafReader(org.apache.lucene.index.LeafReader) FilterLeafReader(org.apache.lucene.index.FilterLeafReader) NamedList(org.apache.solr.common.util.NamedList) IOException(java.io.IOException) SolrIndexSearcher(org.apache.solr.search.SolrIndexSearcher) IOException(java.io.IOException) FieldType(org.apache.solr.schema.FieldType) CollapsingQParserPlugin(org.apache.solr.search.CollapsingQParserPlugin) SchemaField(org.apache.solr.schema.SchemaField) NumberType(org.apache.solr.schema.NumberType) FieldType(org.apache.solr.schema.FieldType) DocValuesType(org.apache.lucene.index.DocValuesType) LongValues(org.apache.lucene.util.LongValues) DocList(org.apache.solr.search.DocList)

Example 12 with DocIterator

use of org.apache.solr.search.DocIterator in project lucene-solr by apache.

the class BasicFunctionalityTest method testLazyField.

@Test
public void testLazyField() throws IOException {
    assertU(adoc("id", "7777", "title", "keyword", "test_hlt", mkstr(10000), "test_hlt", mkstr(20000), "test_hlt", mkstr(30000), "test_hlt", mkstr(40000)));
    assertU(commit());
    SolrCore core = h.getCore();
    // initial request
    SolrQueryRequest req = req("q", "id:7777", "fl", "id,title");
    SolrQueryResponse rsp = new SolrQueryResponse();
    core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp);
    DocList dl = ((ResultContext) rsp.getResponse()).getDocList();
    DocIterator di = dl.iterator();
    Document d1 = req.getSearcher().doc(di.nextDoc());
    IndexableField[] values1 = null;
    // ensure fl field is non lazy, and non-fl field is lazy
    assertFalse(d1.getField("title") instanceof LazyDocument.LazyField);
    assertFalse(d1.getField("id") instanceof LazyDocument.LazyField);
    values1 = d1.getFields("test_hlt");
    assertEquals(4, values1.length);
    for (int i = 0; i < values1.length; i++) {
        assertTrue(values1[i] instanceof LazyDocument.LazyField);
        LazyDocument.LazyField f = (LazyDocument.LazyField) values1[i];
        assertFalse(f.hasBeenLoaded());
    }
    req.close();
    // followup request, different fl
    req = req("q", "id:7777", "fl", "id,test_hlt");
    rsp = new SolrQueryResponse();
    core.execute(core.getRequestHandler(req.getParams().get(CommonParams.QT)), req, rsp);
    dl = ((ResultContext) rsp.getResponse()).getDocList();
    di = dl.iterator();
    Document d2 = req.getSearcher().doc(di.nextDoc());
    // ensure same doc, same lazy field now
    assertTrue("Doc was not cached", d1 == d2);
    IndexableField[] values2 = d2.getFields("test_hlt");
    assertEquals(values1.length, values2.length);
    for (int i = 0; i < values1.length; i++) {
        assertSame("LazyField wasn't reused", values1[i], values2[i]);
        LazyDocument.LazyField f = (LazyDocument.LazyField) values1[i];
        // still not a real boy, no response writer in play
        assertFalse(f.hasBeenLoaded());
    }
    // actuallize one value
    assertNotNull(values2[0].stringValue());
    for (int i = 0; i < values2.length; i++) {
        // now all values for this field should be loaded & cached
        LazyDocument.LazyField f = (LazyDocument.LazyField) values2[i];
        assertTrue(f.hasBeenLoaded());
    }
    req.close();
}
Also used : ResultContext(org.apache.solr.response.ResultContext) SolrQueryResponse(org.apache.solr.response.SolrQueryResponse) DocIterator(org.apache.solr.search.DocIterator) SolrCore(org.apache.solr.core.SolrCore) LazyDocument(org.apache.lucene.document.LazyDocument) Document(org.apache.lucene.document.Document) IndexableField(org.apache.lucene.index.IndexableField) SolrQueryRequest(org.apache.solr.request.SolrQueryRequest) LocalSolrQueryRequest(org.apache.solr.request.LocalSolrQueryRequest) LazyDocument(org.apache.lucene.document.LazyDocument) DocList(org.apache.solr.search.DocList) Test(org.junit.Test)

Example 13 with DocIterator

use of org.apache.solr.search.DocIterator in project ORCID-Source by ORCID.

the class OrcidProfileResponseWriter method write.

@Override
public void write(Writer writer, SolrQueryRequest request, SolrQueryResponse response) throws IOException {
    ResultContext resultContext = (ResultContext) response.getValues().get("response");
    DocIterator iterator = resultContext.docs.iterator();
    if (iterator.hasNext()) {
        int docId = iterator.nextDoc();
        Document doc = request.getSearcher().doc(docId);
        IndexableField field = doc.getField("public-profile-message");
        writer.append(field.stringValue());
    } else
        throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "No record found for reponse writer");
}
Also used : ResultContext(org.apache.solr.response.ResultContext) IndexableField(org.apache.lucene.index.IndexableField) DocIterator(org.apache.solr.search.DocIterator) Document(org.apache.lucene.document.Document) SolrException(org.apache.solr.common.SolrException)

Example 14 with DocIterator

use of org.apache.solr.search.DocIterator in project lucene-solr by apache.

the class ResponseLogComponent method processScores.

protected void processScores(ResponseBuilder rb, DocList dl, IndexSchema schema, SolrIndexSearcher searcher) throws IOException {
    StringBuilder sb = new StringBuilder();
    Set<String> fields = Collections.singleton(schema.getUniqueKeyField().getName());
    for (DocIterator iter = dl.iterator(); iter.hasNext(); ) {
        sb.append(schema.printableUniqueKey(searcher.doc(iter.nextDoc(), fields))).append(':').append(iter.score()).append(',');
    }
    if (sb.length() > 0) {
        rb.rsp.addToLog("responseLog", sb.substring(0, sb.length() - 1));
    }
}
Also used : DocIterator(org.apache.solr.search.DocIterator)

Example 15 with DocIterator

use of org.apache.solr.search.DocIterator in project lucene-solr by apache.

the class StatsField method computeLocalValueSourceStats.

private StatsValues computeLocalValueSourceStats(DocSet base) throws IOException {
    IndexSchema schema = searcher.getSchema();
    final StatsValues allstats = StatsValuesFactory.createStatsValues(this);
    List<FieldFacetStats> facetStats = new ArrayList<>();
    for (String facetField : facets) {
        SchemaField fsf = schema.getField(facetField);
        if (fsf.multiValued()) {
            throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Stats can only facet on single-valued fields, not: " + facetField);
        }
        facetStats.add(new FieldFacetStats(searcher, fsf, this));
    }
    final Iterator<LeafReaderContext> ctxIt = searcher.getIndexReader().leaves().iterator();
    LeafReaderContext ctx = null;
    for (DocIterator docsIt = base.iterator(); docsIt.hasNext(); ) {
        final int doc = docsIt.nextDoc();
        if (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc()) {
            // advance
            do {
                ctx = ctxIt.next();
            } while (ctx == null || doc >= ctx.docBase + ctx.reader().maxDoc());
            assert doc >= ctx.docBase;
            // propagate the context among accumulators.
            allstats.setNextReader(ctx);
            for (FieldFacetStats f : facetStats) {
                f.setNextReader(ctx);
            }
        }
        // accumulate
        allstats.accumulate(doc - ctx.docBase);
        for (FieldFacetStats f : facetStats) {
            f.facet(doc - ctx.docBase);
        }
    }
    for (FieldFacetStats f : facetStats) {
        allstats.addFacet(f.name, f.facetStatsValues);
    }
    return allstats;
}
Also used : SchemaField(org.apache.solr.schema.SchemaField) DocIterator(org.apache.solr.search.DocIterator) ArrayList(java.util.ArrayList) LeafReaderContext(org.apache.lucene.index.LeafReaderContext) IndexSchema(org.apache.solr.schema.IndexSchema) SolrException(org.apache.solr.common.SolrException)

Aggregations

DocIterator (org.apache.solr.search.DocIterator)27 LeafReaderContext (org.apache.lucene.index.LeafReaderContext)10 SchemaField (org.apache.solr.schema.SchemaField)9 DocList (org.apache.solr.search.DocList)8 NamedList (org.apache.solr.common.util.NamedList)7 FieldType (org.apache.solr.schema.FieldType)7 SolrIndexSearcher (org.apache.solr.search.SolrIndexSearcher)7 Document (org.apache.lucene.document.Document)6 IOException (java.io.IOException)5 SolrParams (org.apache.solr.common.params.SolrParams)5 SolrQueryRequest (org.apache.solr.request.SolrQueryRequest)5 ArrayList (java.util.ArrayList)4 IndexableField (org.apache.lucene.index.IndexableField)4 SortedNumericDocValues (org.apache.lucene.index.SortedNumericDocValues)4 FixedBitSet (org.apache.lucene.util.FixedBitSet)4 NumericDocValues (org.apache.lucene.index.NumericDocValues)3 Query (org.apache.lucene.search.Query)3 Sort (org.apache.lucene.search.Sort)3 SolrException (org.apache.solr.common.SolrException)3 SimpleOrderedMap (org.apache.solr.common.util.SimpleOrderedMap)3