Examples with SolrDocumentFetcher org.apache.solr.search.SolrDocumentFetcher used on opensource projects

Example 1 with SolrDocumentFetcher

use of org.apache.solr.search.SolrDocumentFetcher in project lucene-solr by apache.

the class RealTimeGetComponent method getInputDocument.

/**
   * Obtains the latest document for a given id from the tlog or through the realtime searcher (if not found in the tlog). 
   * @param versionReturned If a non-null AtomicLong is passed in, it is set to the version of the update returned from the TLog.
   * @param avoidRetrievingStoredFields Setting this to true avoids fetching stored fields through the realtime searcher,
   *                  however has no effect on documents obtained from the tlog. 
   *                  Non-stored docValues fields are populated anyway, and are not affected by this parameter. Note that if
   *                  the id field is a stored field, it will not be populated if this parameter is true and the document is
   *                  obtained from the index.
   * @param onlyTheseNonStoredDVs If not-null, populate only these DV fields in the document fetched through the realtime searcher. 
   *                  If this is null, decorate all non-stored  DVs (that are not targets of copy fields) from the searcher.
   * @param resolveFullDocument In case the document is fetched from the tlog, it could only be a partial document if the last update
   *                  was an in-place update. In that case, should this partial document be resolved to a full document (by following
   *                  back prevPointer/prevVersion)?
   */
public static SolrInputDocument getInputDocument(SolrCore core, BytesRef idBytes, AtomicLong versionReturned, boolean avoidRetrievingStoredFields, Set<String> onlyTheseNonStoredDVs, boolean resolveFullDocument) throws IOException {
    SolrInputDocument sid = null;
    RefCounted<SolrIndexSearcher> searcherHolder = null;
    try {
        SolrIndexSearcher searcher = null;
        sid = getInputDocumentFromTlog(core, idBytes, versionReturned, onlyTheseNonStoredDVs, resolveFullDocument);
        if (sid == DELETED) {
            return null;
        }
        if (sid == null) {
            // didn't find it in the update log, so it should be in the newest searcher opened
            if (searcher == null) {
                searcherHolder = core.getRealtimeSearcher();
                searcher = searcherHolder.get();
            }
            // SolrCore.verbose("RealTimeGet using searcher ", searcher);
            SchemaField idField = core.getLatestSchema().getUniqueKeyField();
            int docid = searcher.getFirstMatch(new Term(idField.getName(), idBytes));
            if (docid < 0)
                return null;
            SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
            if (avoidRetrievingStoredFields) {
                sid = new SolrInputDocument();
            } else {
                Document luceneDocument = docFetcher.doc(docid);
                sid = toSolrInputDocument(luceneDocument, core.getLatestSchema());
            }
            if (onlyTheseNonStoredDVs != null) {
                docFetcher.decorateDocValueFields(sid, docid, onlyTheseNonStoredDVs);
            } else {
                docFetcher.decorateDocValueFields(sid, docid, docFetcher.getNonStoredDVsWithoutCopyTargets());
            }
        }
    } finally {
        if (searcherHolder != null) {
            searcherHolder.decref();
        }
    }
    if (versionReturned != null) {
        if (sid.containsKey(VERSION_FIELD)) {
            versionReturned.set((long) sid.getFieldValue(VERSION_FIELD));
        }
    }
    return sid;
}

Example 2 with SolrDocumentFetcher

use of org.apache.solr.search.SolrDocumentFetcher in project lucene-solr by apache.

the class RealTimeGetComponent method reopenRealtimeSearcherAndGet.

/**
   * Re-open the RT searcher and get the document, referred to by the idTerm, from that searcher. 
   * @return Returns the document or null if not found.
   */
private static SolrDocument reopenRealtimeSearcherAndGet(SolrCore core, Term idTerm, ReturnFields returnFields) throws IOException {
    UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
    ulog.openRealtimeSearcher();
    RefCounted<SolrIndexSearcher> searcherHolder = core.getRealtimeSearcher();
    try {
        SolrIndexSearcher searcher = searcherHolder.get();
        int docid = searcher.getFirstMatch(idTerm);
        if (docid < 0) {
            return null;
        }
        Document luceneDocument = searcher.doc(docid, returnFields.getLuceneFieldNames());
        SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
        SolrDocumentFetcher docFetcher = searcher.getDocFetcher();
        docFetcher.decorateDocValueFields(doc, docid, docFetcher.getNonStoredDVs(false));
        return doc;
    } finally {
        searcherHolder.decref();
    }
}

Example 3 with SolrDocumentFetcher

use of org.apache.solr.search.SolrDocumentFetcher in project lucene-solr by apache.

the class RealTimeGetComponent method process.

@Override
public void process(ResponseBuilder rb) throws IOException {
    SolrQueryRequest req = rb.req;
    SolrQueryResponse rsp = rb.rsp;
    SolrParams params = req.getParams();
    CloudDescriptor cloudDesc = req.getCore().getCoreDescriptor().getCloudDescriptor();
    if (cloudDesc != null) {
        Replica.Type replicaType = cloudDesc.getReplicaType();
        if (replicaType != null) {
            if (replicaType == Replica.Type.PULL) {
                throw new SolrException(ErrorCode.BAD_REQUEST, String.format(Locale.ROOT, "%s can't handle realtime get requests. Replicas of type %s do not support these type of requests", cloudDesc.getCoreNodeName(), Replica.Type.PULL));
            }
        // non-leader TLOG replicas should not respond to distrib /get requests, but internal requests are OK
        }
    }
    if (!params.getBool(COMPONENT_NAME, true)) {
        return;
    }
    // This seems rather kludgey, may there is better way to indicate
    // that replica can support handling version ranges
    String val = params.get("checkCanHandleVersionRanges");
    if (val != null) {
        rb.rsp.add("canHandleVersionRanges", true);
        return;
    }
    val = params.get("getFingerprint");
    if (val != null) {
        processGetFingeprint(rb);
        return;
    }
    val = params.get("getVersions");
    if (val != null) {
        processGetVersions(rb);
        return;
    }
    val = params.get("getUpdates");
    if (val != null) {
        // solrcloud_debug
        if (log.isDebugEnabled()) {
            try {
                RefCounted<SolrIndexSearcher> searchHolder = req.getCore().getNewestSearcher(false);
                SolrIndexSearcher searcher = searchHolder.get();
                try {
                    log.debug(req.getCore().getCoreContainer().getZkController().getNodeName() + " min count to sync to (from most recent searcher view) " + searcher.search(new MatchAllDocsQuery(), 1).totalHits);
                } finally {
                    searchHolder.decref();
                }
            } catch (Exception e) {
                log.debug("Error in solrcloud_debug block", e);
            }
        }
        processGetUpdates(rb);
        return;
    }
    val = params.get("getInputDocument");
    if (val != null) {
        processGetInputDocument(rb);
        return;
    }
    final IdsRequsted reqIds = IdsRequsted.parseParams(req);
    if (reqIds.allIds.isEmpty()) {
        return;
    }
    // parse any existing filters
    try {
        String[] fqs = req.getParams().getParams(CommonParams.FQ);
        if (fqs != null && fqs.length != 0) {
            List<Query> filters = rb.getFilters();
            // if filters already exists, make a copy instead of modifying the original
            filters = filters == null ? new ArrayList<Query>(fqs.length) : new ArrayList<>(filters);
            for (String fq : fqs) {
                if (fq != null && fq.trim().length() != 0) {
                    QParser fqp = QParser.getParser(fq, req);
                    filters.add(fqp.getQuery());
                }
            }
            if (!filters.isEmpty()) {
                rb.setFilters(filters);
            }
        }
    } catch (SyntaxError e) {
        throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
    }
    final SolrCore core = req.getCore();
    SchemaField idField = core.getLatestSchema().getUniqueKeyField();
    FieldType fieldType = idField.getType();
    SolrDocumentList docList = new SolrDocumentList();
    UpdateLog ulog = core.getUpdateHandler().getUpdateLog();
    SearcherInfo searcherInfo = new SearcherInfo(core);
    // this is initialized & set on the context *after* any searcher (re-)opening
    ResultContext resultContext = null;
    final DocTransformer transformer = rsp.getReturnFields().getTransformer();
    // true in any situation where we have to use a realtime searcher rather then returning docs
    // directly from the UpdateLog
    final boolean mustUseRealtimeSearcher = // if we have filters, we need to check those against the indexed form of the doc
    (rb.getFilters() != null) || ((null != transformer) && transformer.needsSolrIndexSearcher());
    try {
        BytesRefBuilder idBytes = new BytesRefBuilder();
        for (String idStr : reqIds.allIds) {
            fieldType.readableToIndexed(idStr, idBytes);
            if (ulog != null) {
                Object o = ulog.lookup(idBytes.get());
                if (o != null) {
                    // should currently be a List<Oper,Ver,Doc/Id>
                    List entry = (List) o;
                    assert entry.size() >= 3;
                    int oper = (Integer) entry.get(UpdateLog.FLAGS_IDX) & UpdateLog.OPERATION_MASK;
                    switch(oper) {
                        // fall through to ADD
                        case UpdateLog.UPDATE_INPLACE:
                        case UpdateLog.ADD:
                            if (mustUseRealtimeSearcher) {
                                // close handles to current searchers & result context
                                searcherInfo.clear();
                                resultContext = null;
                                // force open a new realtime searcher
                                ulog.openRealtimeSearcher();
                                // pretend we never found this record and fall through to use the searcher
                                o = null;
                                break;
                            }
                            SolrDocument doc;
                            if (oper == UpdateLog.ADD) {
                                doc = toSolrDoc((SolrInputDocument) entry.get(entry.size() - 1), core.getLatestSchema());
                            } else if (oper == UpdateLog.UPDATE_INPLACE) {
                                assert entry.size() == 5;
                                // For in-place update case, we have obtained the partial document till now. We need to
                                // resolve it to a full document to be returned to the user.
                                doc = resolveFullDocument(core, idBytes.get(), rsp.getReturnFields(), (SolrInputDocument) entry.get(entry.size() - 1), entry, null);
                                if (doc == null) {
                                    // document has been deleted as the resolve was going on
                                    break;
                                }
                            } else {
                                throw new SolrException(ErrorCode.INVALID_STATE, "Expected ADD or UPDATE_INPLACE. Got: " + oper);
                            }
                            if (transformer != null) {
                                // unknown docID
                                transformer.transform(doc, -1, 0);
                            }
                            docList.add(doc);
                            break;
                        case UpdateLog.DELETE:
                            break;
                        default:
                            throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "Unknown Operation! " + oper);
                    }
                    if (o != null)
                        continue;
                }
            }
            // didn't find it in the update log, so it should be in the newest searcher opened
            searcherInfo.init();
            // don't bother with ResultContext yet, we won't need it if doc doesn't match filters
            int docid = -1;
            long segAndId = searcherInfo.getSearcher().lookupId(idBytes.get());
            if (segAndId >= 0) {
                int segid = (int) segAndId;
                LeafReaderContext ctx = searcherInfo.getSearcher().getTopReaderContext().leaves().get((int) (segAndId >> 32));
                docid = segid + ctx.docBase;
                if (rb.getFilters() != null) {
                    for (Query raw : rb.getFilters()) {
                        Query q = raw.rewrite(searcherInfo.getSearcher().getIndexReader());
                        Scorer scorer = searcherInfo.getSearcher().createWeight(q, false, 1f).scorer(ctx);
                        if (scorer == null || segid != scorer.iterator().advance(segid)) {
                            // filter doesn't match.
                            docid = -1;
                            break;
                        }
                    }
                }
            }
            if (docid < 0)
                continue;
            Document luceneDocument = searcherInfo.getSearcher().doc(docid, rsp.getReturnFields().getLuceneFieldNames());
            SolrDocument doc = toSolrDoc(luceneDocument, core.getLatestSchema());
            SolrDocumentFetcher docFetcher = searcherInfo.getSearcher().getDocFetcher();
            docFetcher.decorateDocValueFields(doc, docid, docFetcher.getNonStoredDVs(true));
            if (null != transformer) {
                if (null == resultContext) {
                    // either first pass, or we've re-opened searcher - either way now we setContext
                    resultContext = new RTGResultContext(rsp.getReturnFields(), searcherInfo.getSearcher(), req);
                    transformer.setContext(resultContext);
                }
                transformer.transform(doc, docid, 0);
            }
            docList.add(doc);
        }
    } finally {
        searcherInfo.clear();
    }
    addDocListToResponse(rb, docList);
}