Examples with SolrIndexSearcher org.apache.solr.search.SolrIndexSearcher used on opensource projects

Example 61 with SolrIndexSearcher

use of org.apache.solr.search.SolrIndexSearcher in project lucene-solr by apache.

the class ResponseLogComponent method process.

@Override
public void process(ResponseBuilder rb) throws IOException {
    SolrParams params = rb.req.getParams();
    if (!params.getBool(COMPONENT_NAME, false))
        return;
    SolrIndexSearcher searcher = rb.req.getSearcher();
    IndexSchema schema = searcher.getSchema();
    if (schema.getUniqueKeyField() == null)
        return;
    ResultContext rc = (ResultContext) rb.rsp.getResponse();
    DocList docs = rc.getDocList();
    if (docs.hasScores()) {
        processScores(rb, docs, schema, searcher);
    } else {
        processIds(rb, docs, schema, searcher);
    }
}

Example 62 with SolrIndexSearcher

use of org.apache.solr.search.SolrIndexSearcher in project lucene-solr by apache.

the class FieldOptions method process.

@Override
public void process(ResponseBuilder rb) throws IOException {
    SolrParams params = rb.req.getParams();
    if (!params.getBool(COMPONENT_NAME, false)) {
        return;
    }
    NamedList<Object> termVectors = new NamedList<>();
    rb.rsp.add(TERM_VECTORS, termVectors);
    IndexSchema schema = rb.req.getSchema();
    SchemaField keyField = schema.getUniqueKeyField();
    String uniqFieldName = null;
    if (keyField != null) {
        uniqFieldName = keyField.getName();
    }
    FieldOptions allFields = new FieldOptions();
    //figure out what options we have, and try to get the appropriate vector
    allFields.termFreq = params.getBool(TermVectorParams.TF, false);
    allFields.positions = params.getBool(TermVectorParams.POSITIONS, false);
    allFields.offsets = params.getBool(TermVectorParams.OFFSETS, false);
    allFields.payloads = params.getBool(TermVectorParams.PAYLOADS, false);
    allFields.docFreq = params.getBool(TermVectorParams.DF, false);
    allFields.tfIdf = params.getBool(TermVectorParams.TF_IDF, false);
    //short cut to all values.
    if (params.getBool(TermVectorParams.ALL, false)) {
        allFields.termFreq = true;
        allFields.positions = true;
        allFields.offsets = true;
        allFields.payloads = true;
        allFields.docFreq = true;
        allFields.tfIdf = true;
    }
    //Build up our per field mapping
    Map<String, FieldOptions> fieldOptions = new HashMap<>();
    NamedList<List<String>> warnings = new NamedList<>();
    List<String> noTV = new ArrayList<>();
    List<String> noPos = new ArrayList<>();
    List<String> noOff = new ArrayList<>();
    List<String> noPay = new ArrayList<>();
    Set<String> fields = getFields(rb);
    if (null != fields) {
        //we have specific fields to retrieve, or no fields
        for (String field : fields) {
            // workaround SOLR-3523
            if (null == field || "score".equals(field))
                continue;
            // we don't want to issue warnings about the uniqueKey field
            // since it can cause lots of confusion in distributed requests
            // where the uniqueKey field is injected into the fl for merging
            final boolean fieldIsUniqueKey = field.equals(uniqFieldName);
            SchemaField sf = schema.getFieldOrNull(field);
            if (sf != null) {
                if (sf.storeTermVector()) {
                    FieldOptions option = fieldOptions.get(field);
                    if (option == null) {
                        option = new FieldOptions();
                        option.fieldName = field;
                        fieldOptions.put(field, option);
                    }
                    //get the per field mappings
                    option.termFreq = params.getFieldBool(field, TermVectorParams.TF, allFields.termFreq);
                    option.docFreq = params.getFieldBool(field, TermVectorParams.DF, allFields.docFreq);
                    option.tfIdf = params.getFieldBool(field, TermVectorParams.TF_IDF, allFields.tfIdf);
                    //Validate these are even an option
                    option.positions = params.getFieldBool(field, TermVectorParams.POSITIONS, allFields.positions);
                    if (option.positions && !sf.storeTermPositions() && !fieldIsUniqueKey) {
                        noPos.add(field);
                    }
                    option.offsets = params.getFieldBool(field, TermVectorParams.OFFSETS, allFields.offsets);
                    if (option.offsets && !sf.storeTermOffsets() && !fieldIsUniqueKey) {
                        noOff.add(field);
                    }
                    option.payloads = params.getFieldBool(field, TermVectorParams.PAYLOADS, allFields.payloads);
                    if (option.payloads && !sf.storeTermPayloads() && !fieldIsUniqueKey) {
                        noPay.add(field);
                    }
                } else {
                    //field doesn't have term vectors
                    if (!fieldIsUniqueKey)
                        noTV.add(field);
                }
            } else {
                //field doesn't exist
                throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "undefined field: " + field);
            }
        }
    }
    // to be changed to account for that.
    if (!noTV.isEmpty()) {
        warnings.add("noTermVectors", noTV);
    }
    if (!noPos.isEmpty()) {
        warnings.add("noPositions", noPos);
    }
    if (!noOff.isEmpty()) {
        warnings.add("noOffsets", noOff);
    }
    if (!noPay.isEmpty()) {
        warnings.add("noPayloads", noPay);
    }
    if (warnings.size() > 0) {
        termVectors.add(TV_KEY_WARNINGS, warnings);
    }
    DocListAndSet listAndSet = rb.getResults();
    List<Integer> docIds = getInts(params.getParams(TermVectorParams.DOC_IDS));
    Iterator<Integer> iter;
    if (docIds != null && !docIds.isEmpty()) {
        iter = docIds.iterator();
    } else {
        DocList list = listAndSet.docList;
        iter = list.iterator();
    }
    SolrIndexSearcher searcher = rb.req.getSearcher();
    IndexReader reader = searcher.getIndexReader();
    //the TVMapper is a TermVectorMapper which can be used to optimize loading of Term Vectors
    //Only load the id field to get the uniqueKey of that
    //field
    final String finalUniqFieldName = uniqFieldName;
    final List<String> uniqValues = new ArrayList<>();
    // TODO: is this required to be single-valued? if so, we should STOP
    // once we find it...
    final StoredFieldVisitor getUniqValue = new StoredFieldVisitor() {

        @Override
        public void stringField(FieldInfo fieldInfo, byte[] bytes) {
            uniqValues.add(new String(bytes, StandardCharsets.UTF_8));
        }

        @Override
        public void intField(FieldInfo fieldInfo, int value) {
            uniqValues.add(Integer.toString(value));
        }

        @Override
        public void longField(FieldInfo fieldInfo, long value) {
            uniqValues.add(Long.toString(value));
        }

        @Override
        public Status needsField(FieldInfo fieldInfo) {
            return (fieldInfo.name.equals(finalUniqFieldName)) ? Status.YES : Status.NO;
        }
    };
    while (iter.hasNext()) {
        Integer docId = iter.next();
        NamedList<Object> docNL = new NamedList<>();
        if (keyField != null) {
            reader.document(docId, getUniqValue);
            String uniqVal = null;
            if (uniqValues.size() != 0) {
                uniqVal = uniqValues.get(0);
                uniqValues.clear();
                docNL.add("uniqueKey", uniqVal);
                termVectors.add(uniqVal, docNL);
            }
        } else {
            // support for schemas w/o a unique key,
            termVectors.add("doc-" + docId, docNL);
        }
        if (null != fields) {
            for (Map.Entry<String, FieldOptions> entry : fieldOptions.entrySet()) {
                final String field = entry.getKey();
                final Terms vector = reader.getTermVector(docId, field);
                if (vector != null) {
                    TermsEnum termsEnum = vector.iterator();
                    mapOneVector(docNL, entry.getValue(), reader, docId, termsEnum, field);
                }
            }
        } else {
            // extract all fields
            final Fields vectors = reader.getTermVectors(docId);
            for (String field : vectors) {
                Terms terms = vectors.terms(field);
                if (terms != null) {
                    TermsEnum termsEnum = terms.iterator();
                    mapOneVector(docNL, allFields, reader, docId, termsEnum, field);
                }
            }
        }
    }
}

Example 63 with SolrIndexSearcher

use of org.apache.solr.search.SolrIndexSearcher in project lucene-solr by apache.

the class RealTimeGetComponent method mergePartialDocWithFullDocFromIndex.

/**
   * Gets a document from the index by id. If a non-null partial document (for in-place update) is passed in,
   * this method obtains the document from the tlog/index by the given id, merges the partial document on top of it and then returns
   * the resultant document.
   *
   * @param core           A SolrCore instance, useful for obtaining a realtimesearcher and the schema
   * @param idBytes        Binary representation of the value of the unique key field
   * @param returnFields   Return fields, as requested
   * @param onlyTheseFields When a non-null set of field names is passed in, the merge process only attempts to merge
   *        the given fields in this set. When this set is null, it merges all fields.
   * @param partialDoc     A partial document (containing an in-place update) used for merging against a full document
   *                       from index; this maybe be null.
   * @return If partial document is null, this returns document from the index or null if not found. 
   *         If partial document is not null, this returns a document from index merged with the partial document, or null if
   *         document doesn't exist in the index.
   */
private static SolrDocument mergePartialDocWithFullDocFromIndex(SolrCore core, BytesRef idBytes, ReturnFields returnFields, Set<String> onlyTheseFields, SolrInputDocument partialDoc) throws IOException {
    //Searcher();
    RefCounted<SolrIndexSearcher> searcherHolder = core.getRealtimeSearcher();
    try {
        // now fetch last document from index, and merge partialDoc on top of it
        SolrIndexSearcher searcher = searcherHolder.get();
        SchemaField idField = core.getLatestSchema().getUniqueKeyField();
        Term idTerm = new Term(idField.getName(), idBytes);
        int docid = searcher.getFirstMatch(idTerm);
        if (docid < 0) {
            // The document was not found in index! Reopen a new RT searcher (to be sure) and get again.
            // This should be because the document was deleted recently.
            SolrDocument doc = reopenRealtimeSearcherAndGet(core, idTerm, returnFields);
            if (doc == null) {
                // This must be a case of deleted doc
                return null;
            }
            return doc;
        }
        SolrDocument doc;
        Set<String> decorateFields = onlyTheseFields == null ? searcher.getDocFetcher().getNonStoredDVs(false) : onlyTheseFields;
        Document luceneDocument = searcher.doc(docid, returnFields.getLuceneFieldNames());
        doc = toSolrDoc(luceneDocument, core.getLatestSchema());
        searcher.getDocFetcher().decorateDocValueFields(doc, docid, decorateFields);
        long docVersion = (long) doc.getFirstValue(VERSION_FIELD);
        Object partialVersionObj = partialDoc.getFieldValue(VERSION_FIELD);
        long partialDocVersion = partialVersionObj instanceof Field ? ((Field) partialVersionObj).numericValue().longValue() : partialVersionObj instanceof Number ? ((Number) partialVersionObj).longValue() : Long.parseLong(partialVersionObj.toString());
        if (docVersion > partialDocVersion) {
            return doc;
        }
        for (String fieldName : partialDoc.getFieldNames()) {
            // since partial doc will only contain single valued fields, this is fine
            doc.setField(fieldName.toString(), partialDoc.getFieldValue(fieldName));
        }
        return doc;
    } finally {
        if (searcherHolder != null) {
            searcherHolder.decref();
        }
    }
}

Example 64 with SolrIndexSearcher

use of org.apache.solr.search.SolrIndexSearcher in project lucene-solr by apache.

the class RealTimeGetComponent method process.

@Override
public void process(ResponseBuilder rb) throws IOException {
    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;
    SolrParams params = req.getParams();
    CloudDescriptor cloudDesc = req.getCore().getCoreDescriptor().getCloudDescriptor();
    if (cloudDesc != null) {
        Replica.Type replicaType = cloudDesc.getReplicaType();
        if (replicaType != null) {
            if (replicaType == Replica.Type.PULL) {
                throw new SolrException(ErrorCode.BAD_REQUEST, String.format(Locale.ROOT, "%s can't handle realtime get requests. Replicas of type %s do not support these type of requests", cloudDesc.getCoreNodeName(), Replica.Type.PULL));
            }
        // non-leader TLOG replicas should not respond to distrib /get requests, but internal requests are OK
        }
    }
    if (!params.getBool(COMPONENT_NAME, true)) {
        return;
    }
    // This seems rather kludgey, may there is better way to indicate
    // that replica can support handling version ranges
    String val = params.get("checkCanHandleVersionRanges");
    if (val != null) {
        rb.rsp.add("canHandleVersionRanges", true);
        return;
    }
    val = params.get("getFingerprint");
    if (val != null) {
        processGetFingeprint(rb);
        return;
    }
    val = params.get("getVersions");
    if (val != null) {
        processGetVersions(rb);
        return;
    }
    val = params.get("getUpdates");
    if (val != null) {
        // solrcloud_debug
        if (log.isDebugEnabled()) {
            try {
                RefCounted<SolrIndexSearcher> searchHolder = req.getCore().getNewestSearcher(false);
                SolrIndexSearcher searcher = searchHolder.get();
                try {
                    log.debug(req.getCore().getCoreContainer().getZkController().getNodeName() + " min count to sync to (from most recent searcher view) " + searcher.search(new MatchAllDocsQuery(), 1).totalHits);
                } finally {
                    searchHolder.decref();
                }
            } catch (Exception e) {
                log.debug("Error in solrcloud_debug block", e);
            }
        }
        processGetUpdates(rb);
        return;
    }
    val = params.get("getInputDocument");
    if (val != null) {
        processGetInputDocument(rb);
        return;
    }
    final IdsRequsted reqIds = IdsRequsted.parseParams(req);
    if (reqIds.allIds.isEmpty()) {
        return;
    }
    // parse any existing filters
    try {
        String[] fqs = req.getParams().getParams(CommonParams.FQ);
        if (fqs != null && fqs.length != 0) {
            List<Query> filters = rb.getFilters();
            // if filters already exists, make a copy instead of modifying the original
            filters = filters == null ? new ArrayList<Query>(fqs.length) : new ArrayList<>(filters);
            for (String fq : fqs) {
                if (fq != null && fq.trim().length() != 0) {
                    QParser fqp = QParser.getParser(fq, req);
                    filters.add(fqp.getQuery());
                }
            }
            if (!filters.isEmpty()) {
                rb.setFilters(filters);
            }
        }
    } catch (SyntaxError e) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
    }
    final SolrCore core = req.getCore();
    SchemaField idField = core.getLatestSchema().getUniqueKeyField();
    FieldType fieldType = idField.getType();
    SolrDocumentList docList = new SolrDocumentList();
    UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
    SearcherInfo searcherInfo = new SearcherInfo(core);
    // this is initialized & set on the context *after* any searcher (re-)opening
    ResultContext resultContext = null;
    final DocTransformer transformer = rsp.getReturnFields().getTransformer();
    // true in any situation where we have to use a realtime searcher rather then returning docs
    // directly from the UpdateLog
    final boolean mustUseRealtimeSearcher = // if we have filters, we need to check those against the indexed form of the doc
    (rb.getFilters() != null) || ((null != transformer) && transformer.needsSolrIndexSearcher());
    try {
        BytesRefBuilder idBytes = new BytesRefBuilder();
        for (String idStr : reqIds.allIds) {
            fieldType.readableToIndexed(idStr, idBytes);
            if (ulog != null) {
                Object o = ulog.lookup(idBytes.get());
                if (o != null) {
                    // should currently be a List<Oper,Ver,Doc/Id>
                    List entry = (List) o;
                    assert entry.size() >= 3;
                    int oper = (Integer) entry.get(UpdateLog.FLAGS_IDX) & UpdateLog.OPERATION_MASK;
                    switch(oper) {
                        // fall through to ADD
                        case UpdateLog.UPDATE_INPLACE:
                        case UpdateLog.ADD:
                            if (mustUseRealtimeSearcher) {
                                // close handles to current searchers & result context
                                searcherInfo.clear();
                                resultContext = null;
                                // force open a new realtime searcher
                                ulog.openRealtimeSearcher();
                                // pretend we never found this record and fall through to use the searcher
                                o = null;
                                break;
                            }
                            SolrDocument doc;
                            if (oper == UpdateLog.ADD) {
                                doc = toSolrDoc((SolrInputDocument) entry.get(entry.size() - 1), core.getLatestSchema());
                            } else if (oper == UpdateLog.UPDATE_INPLACE) {
                                assert entry.size() == 5;
                                // For in-place update case, we have obtained the partial document till now. We need to
                                // resolve it to a full document to be returned to the user.
                                doc = resolveFullDocument(core, idBytes.get(), rsp.getReturnFields(), (SolrInputDocument) entry.get(entry.size() - 1), entry, null);
                                if (doc == null) {
                                    // document has been deleted as the resolve was going on
                                    break;
                                }
                            } else {
                                throw new SolrException(ErrorCode.INVALID_STATE, "Expected ADD or UPDATE_INPLACE. Got: " + oper);
                            }
                            if (transformer != null) {
                                // unknown docID
                                transformer.transform(doc, -1, 0);
                            }
                            docList.add(doc);
                            break;
                        case UpdateLog.DELETE:
                            break;
                        default:
                            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown Operation! " + oper);
                    }
                    if (o != null)
                        continue;
                }
            }
            // didn't find it in the update log, so it should be in the newest searcher opened
            searcherInfo.init();
            // don't bother with ResultContext yet, we won't need it if doc doesn't match filters
            int docid = -1;
            long segAndId = searcherInfo.getSearcher().lookupId(idBytes.get());
            if (segAndId >= 0) {
                int segid = (int) segAndId;
                LeafReaderContext ctx = searcherInfo.getSearcher().getTopReaderContext().leaves().get((int) (segAndId >> 32));
                docid = segid + ctx.docBase;
                if (rb.getFilters() != null) {
                    for (Query raw : rb.getFilters()) {
                        Query q = raw.rewrite(searcherInfo.getSearcher().getIndexReader());
                        Scorer scorer = searcherInfo.getSearcher().createWeight(q, false, 1f).scorer(ctx);
                        if (scorer == null || segid != scorer.iterator().advance(segid)) {
                            // filter doesn't match.
                            docid = -1;
                            break;
                        }
                    }
                }
            }
            if (docid < 0)
                continue;
            Document luceneDocument = searcherInfo.getSearcher().doc(docid, rsp.getReturnFields().getLuceneFieldNames());
            SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
            SolrDocumentFetcher docFetcher = searcherInfo.getSearcher().getDocFetcher();
            docFetcher.decorateDocValueFields(doc, docid, docFetcher.getNonStoredDVs(true));
            if (null != transformer) {
                if (null == resultContext) {
                    // either first pass, or we've re-opened searcher - either way now we setContext
                    resultContext = new RTGResultContext(rsp.getReturnFields(), searcherInfo.getSearcher(), req);
                    transformer.setContext(resultContext);
                }
                transformer.transform(doc, docid, 0);
            }
            docList.add(doc);
        }
    } finally {
        searcherInfo.clear();
    }
    addDocListToResponse(rb, docList);
}

Example 65 with SolrIndexSearcher

use of org.apache.solr.search.SolrIndexSearcher in project lucene-solr by apache.

the class SolrPluginUtils method doStandardResultsDebug.

public static void doStandardResultsDebug(SolrQueryRequest req, Query query, DocList results, boolean dbgResults, NamedList dbg) throws IOException {
    if (dbgResults) {
        SolrIndexSearcher searcher = req.getSearcher();
        IndexSchema schema = searcher.getSchema();
        boolean explainStruct = req.getParams().getBool(CommonParams.EXPLAIN_STRUCT, false);
        if (results != null) {
            NamedList<Explanation> explain = getExplanations(query, results, searcher, schema);
            dbg.add("explain", explainStruct ? explanationsToNamedLists(explain) : explanationsToStrings(explain));
        }
        String otherQueryS = req.getParams().get(CommonParams.EXPLAIN_OTHER);
        if (otherQueryS != null && otherQueryS.length() > 0) {
            DocList otherResults = doSimpleQuery(otherQueryS, req, 0, 10);
            dbg.add("otherQuery", otherQueryS);
            NamedList<Explanation> explainO = getExplanations(query, otherResults, searcher, schema);
            dbg.add("explainOther", explainStruct ? explanationsToNamedLists(explainO) : explanationsToStrings(explainO));
        }
    }
}