Examples with ShardResponse org.apache.solr.handler.component.ShardResponse used on opensource projects

Example 11 with ShardResponse

use of org.apache.solr.handler.component.ShardResponse in project lucene-solr by apache.

the class PeerSync method canHandleVersionRanges.

private boolean canHandleVersionRanges(String replica) {
    SyncShardRequest sreq = new SyncShardRequest();
    requests.add(sreq);
    // determine if leader can handle version ranges
    sreq.shards = new String[] { replica };
    sreq.actualShards = sreq.shards;
    sreq.params = new ModifiableSolrParams();
    sreq.params.set("qt", "/get");
    sreq.params.set(DISTRIB, false);
    sreq.params.set("checkCanHandleVersionRanges", false);
    ShardHandler sh = shardHandlerFactory.getShardHandler(client);
    sh.submit(sreq, replica, sreq.params);
    ShardResponse srsp = sh.takeCompletedIncludingErrors();
    Boolean canHandleVersionRanges = srsp.getSolrResponse().getResponse().getBooleanArg("canHandleVersionRanges");
    if (canHandleVersionRanges == null || canHandleVersionRanges.booleanValue() == false) {
        return false;
    }
    return true;
}

Example 12 with ShardResponse

use of org.apache.solr.handler.component.ShardResponse in project lucene-solr by apache.

the class PeerSync method sync.

/** Requests and applies recent updates from peers */
public static void sync(SolrCore core, List<String> replicas, int nUpdates) {
    ShardHandlerFactory shardHandlerFactory = core.getCoreContainer().getShardHandlerFactory();
    ShardHandler shardHandler = shardHandlerFactory.getShardHandler();
    for (String replica : replicas) {
        ShardRequest sreq = new ShardRequest();
        sreq.shards = new String[] { replica };
        sreq.params = new ModifiableSolrParams();
        sreq.params.set("qt", "/get");
        sreq.params.set(DISTRIB, false);
        sreq.params.set("getVersions", nUpdates);
        shardHandler.submit(sreq, replica, sreq.params);
    }
    for (String replica : replicas) {
        ShardResponse srsp = shardHandler.takeCompletedOrError();
    }
}

Example 13 with ShardResponse

use of org.apache.solr.handler.component.ShardResponse in project lucene-solr by apache.

the class SearchGroupShardResponseProcessor method process.

/**
   * {@inheritDoc}
   */
@Override
public void process(ResponseBuilder rb, ShardRequest shardRequest) {
    SortSpec groupSortSpec = rb.getGroupingSpec().getGroupSortSpec();
    Sort groupSort = rb.getGroupingSpec().getGroupSort();
    final String[] fields = rb.getGroupingSpec().getFields();
    Sort withinGroupSort = rb.getGroupingSpec().getSortWithinGroup();
    assert withinGroupSort != null;
    final Map<String, List<Collection<SearchGroup<BytesRef>>>> commandSearchGroups = new HashMap<>(fields.length, 1.0f);
    final Map<String, Map<SearchGroup<BytesRef>, Set<String>>> tempSearchGroupToShards = new HashMap<>(fields.length, 1.0f);
    for (String field : fields) {
        commandSearchGroups.put(field, new ArrayList<Collection<SearchGroup<BytesRef>>>(shardRequest.responses.size()));
        tempSearchGroupToShards.put(field, new HashMap<SearchGroup<BytesRef>, Set<String>>());
        if (!rb.searchGroupToShards.containsKey(field)) {
            rb.searchGroupToShards.put(field, new HashMap<SearchGroup<BytesRef>, Set<String>>());
        }
    }
    SearchGroupsResultTransformer serializer = new SearchGroupsResultTransformer(rb.req.getSearcher());
    int maxElapsedTime = 0;
    int hitCountDuringFirstPhase = 0;
    NamedList<Object> shardInfo = null;
    if (rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
        shardInfo = new SimpleOrderedMap<>(shardRequest.responses.size());
        rb.rsp.getValues().add(ShardParams.SHARDS_INFO + ".firstPhase", shardInfo);
    }
    for (ShardResponse srsp : shardRequest.responses) {
        if (shardInfo != null) {
            SimpleOrderedMap<Object> nl = new SimpleOrderedMap<>(4);
            if (srsp.getException() != null) {
                Throwable t = srsp.getException();
                if (t instanceof SolrServerException) {
                    t = ((SolrServerException) t).getCause();
                }
                nl.add("error", t.toString());
                StringWriter trace = new StringWriter();
                t.printStackTrace(new PrintWriter(trace));
                nl.add("trace", trace.toString());
            } else {
                nl.add("numFound", (Integer) srsp.getSolrResponse().getResponse().get("totalHitCount"));
            }
            if (srsp.getSolrResponse() != null) {
                nl.add("time", srsp.getSolrResponse().getElapsedTime());
            }
            if (srsp.getShardAddress() != null) {
                nl.add("shardAddress", srsp.getShardAddress());
            }
            shardInfo.add(srsp.getShard(), nl);
        }
        if (rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false) && srsp.getException() != null) {
            if (rb.rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY) == null) {
                rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
            }
            // continue if there was an error and we're tolerant.
            continue;
        }
        maxElapsedTime = (int) Math.max(maxElapsedTime, srsp.getSolrResponse().getElapsedTime());
        @SuppressWarnings("unchecked") NamedList<NamedList> firstPhaseResult = (NamedList<NamedList>) srsp.getSolrResponse().getResponse().get("firstPhase");
        final Map<String, SearchGroupsFieldCommandResult> result = serializer.transformToNative(firstPhaseResult, groupSort, withinGroupSort, srsp.getShard());
        for (String field : commandSearchGroups.keySet()) {
            final SearchGroupsFieldCommandResult firstPhaseCommandResult = result.get(field);
            final Integer groupCount = firstPhaseCommandResult.getGroupCount();
            if (groupCount != null) {
                Integer existingGroupCount = rb.mergedGroupCounts.get(field);
                // Assuming groups don't cross shard boundary...
                rb.mergedGroupCounts.put(field, existingGroupCount != null ? Integer.valueOf(existingGroupCount + groupCount) : groupCount);
            }
            final Collection<SearchGroup<BytesRef>> searchGroups = firstPhaseCommandResult.getSearchGroups();
            if (searchGroups == null) {
                continue;
            }
            commandSearchGroups.get(field).add(searchGroups);
            for (SearchGroup<BytesRef> searchGroup : searchGroups) {
                Map<SearchGroup<BytesRef>, Set<String>> map = tempSearchGroupToShards.get(field);
                Set<String> shards = map.get(searchGroup);
                if (shards == null) {
                    shards = new HashSet<>();
                    map.put(searchGroup, shards);
                }
                shards.add(srsp.getShard());
            }
        }
        hitCountDuringFirstPhase += (Integer) srsp.getSolrResponse().getResponse().get("totalHitCount");
    }
    rb.totalHitCount = hitCountDuringFirstPhase;
    rb.firstPhaseElapsedTime = maxElapsedTime;
    for (String groupField : commandSearchGroups.keySet()) {
        List<Collection<SearchGroup<BytesRef>>> topGroups = commandSearchGroups.get(groupField);
        Collection<SearchGroup<BytesRef>> mergedTopGroups = SearchGroup.merge(topGroups, groupSortSpec.getOffset(), groupSortSpec.getCount(), groupSort);
        if (mergedTopGroups == null) {
            continue;
        }
        rb.mergedSearchGroups.put(groupField, mergedTopGroups);
        for (SearchGroup<BytesRef> mergedTopGroup : mergedTopGroups) {
            rb.searchGroupToShards.get(groupField).put(mergedTopGroup, tempSearchGroupToShards.get(groupField).get(mergedTopGroup));
        }
    }
}

Example 14 with ShardResponse

use of org.apache.solr.handler.component.ShardResponse in project lucene-solr by apache.

the class TopGroupsShardResponseProcessor method process.

/**
   * {@inheritDoc}
   */
@Override
@SuppressWarnings("unchecked")
public void process(ResponseBuilder rb, ShardRequest shardRequest) {
    Sort groupSort = rb.getGroupingSpec().getGroupSort();
    String[] fields = rb.getGroupingSpec().getFields();
    String[] queries = rb.getGroupingSpec().getQueries();
    Sort withinGroupSort = rb.getGroupingSpec().getSortWithinGroup();
    assert withinGroupSort != null;
    // If group.format=simple group.offset doesn't make sense
    int groupOffsetDefault;
    if (rb.getGroupingSpec().getResponseFormat() == Grouping.Format.simple || rb.getGroupingSpec().isMain()) {
        groupOffsetDefault = 0;
    } else {
        groupOffsetDefault = rb.getGroupingSpec().getWithinGroupOffset();
    }
    int docsPerGroupDefault = rb.getGroupingSpec().getWithinGroupLimit();
    Map<String, List<TopGroups<BytesRef>>> commandTopGroups = new HashMap<>();
    for (String field : fields) {
        commandTopGroups.put(field, new ArrayList<TopGroups<BytesRef>>());
    }
    Map<String, List<QueryCommandResult>> commandTopDocs = new HashMap<>();
    for (String query : queries) {
        commandTopDocs.put(query, new ArrayList<QueryCommandResult>());
    }
    TopGroupsResultTransformer serializer = new TopGroupsResultTransformer(rb);
    NamedList<Object> shardInfo = null;
    if (rb.req.getParams().getBool(ShardParams.SHARDS_INFO, false)) {
        shardInfo = new SimpleOrderedMap<>();
        rb.rsp.getValues().add(ShardParams.SHARDS_INFO, shardInfo);
    }
    for (ShardResponse srsp : shardRequest.responses) {
        SimpleOrderedMap<Object> individualShardInfo = null;
        if (shardInfo != null) {
            individualShardInfo = new SimpleOrderedMap<>();
            if (srsp.getException() != null) {
                Throwable t = srsp.getException();
                if (t instanceof SolrServerException && ((SolrServerException) t).getCause() != null) {
                    t = ((SolrServerException) t).getCause();
                }
                individualShardInfo.add("error", t.toString());
                StringWriter trace = new StringWriter();
                t.printStackTrace(new PrintWriter(trace));
                individualShardInfo.add("trace", trace.toString());
            } else {
            // summary for successful shard response is added down below
            }
            if (srsp.getSolrResponse() != null) {
                individualShardInfo.add("time", srsp.getSolrResponse().getElapsedTime());
            }
            if (srsp.getShardAddress() != null) {
                individualShardInfo.add("shardAddress", srsp.getShardAddress());
            }
            shardInfo.add(srsp.getShard(), individualShardInfo);
        }
        if (rb.req.getParams().getBool(ShardParams.SHARDS_TOLERANT, false) && srsp.getException() != null) {
            if (rb.rsp.getResponseHeader().get(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY) == null) {
                rb.rsp.getResponseHeader().add(SolrQueryResponse.RESPONSE_HEADER_PARTIAL_RESULTS_KEY, Boolean.TRUE);
            }
            // continue if there was an error and we're tolerant.  
            continue;
        }
        NamedList<NamedList> secondPhaseResult = (NamedList<NamedList>) srsp.getSolrResponse().getResponse().get("secondPhase");
        if (secondPhaseResult == null)
            continue;
        Map<String, ?> result = serializer.transformToNative(secondPhaseResult, groupSort, withinGroupSort, srsp.getShard());
        int numFound = 0;
        float maxScore = Float.NaN;
        for (String field : commandTopGroups.keySet()) {
            TopGroups<BytesRef> topGroups = (TopGroups<BytesRef>) result.get(field);
            if (topGroups == null) {
                continue;
            }
            if (individualShardInfo != null) {
                // keep track of this when shards.info=true
                numFound += topGroups.totalHitCount;
                if (Float.isNaN(maxScore) || topGroups.maxScore > maxScore)
                    maxScore = topGroups.maxScore;
            }
            commandTopGroups.get(field).add(topGroups);
        }
        for (String query : queries) {
            QueryCommandResult queryCommandResult = (QueryCommandResult) result.get(query);
            if (individualShardInfo != null) {
                // keep track of this when shards.info=true
                numFound += queryCommandResult.getMatches();
                float thisMax = queryCommandResult.getTopDocs().getMaxScore();
                if (Float.isNaN(maxScore) || thisMax > maxScore)
                    maxScore = thisMax;
            }
            commandTopDocs.get(query).add(queryCommandResult);
        }
        if (individualShardInfo != null) {
            // when shards.info=true
            individualShardInfo.add("numFound", numFound);
            individualShardInfo.add("maxScore", maxScore);
        }
    }
    for (String groupField : commandTopGroups.keySet()) {
        List<TopGroups<BytesRef>> topGroups = commandTopGroups.get(groupField);
        if (topGroups.isEmpty()) {
            continue;
        }
        TopGroups<BytesRef>[] topGroupsArr = new TopGroups[topGroups.size()];
        int docsPerGroup = docsPerGroupDefault;
        if (docsPerGroup < 0) {
            docsPerGroup = 0;
            for (TopGroups subTopGroups : topGroups) {
                docsPerGroup += subTopGroups.totalGroupedHitCount;
            }
        }
        rb.mergedTopGroups.put(groupField, TopGroups.merge(topGroups.toArray(topGroupsArr), groupSort, withinGroupSort, groupOffsetDefault, docsPerGroup, TopGroups.ScoreMergeMode.None));
    }
    for (String query : commandTopDocs.keySet()) {
        List<QueryCommandResult> queryCommandResults = commandTopDocs.get(query);
        List<TopDocs> topDocs = new ArrayList<>(queryCommandResults.size());
        int mergedMatches = 0;
        for (QueryCommandResult queryCommandResult : queryCommandResults) {
            topDocs.add(queryCommandResult.getTopDocs());
            mergedMatches += queryCommandResult.getMatches();
        }
        int topN = rb.getGroupingSpec().getOffset() + rb.getGroupingSpec().getLimit();
        final TopDocs mergedTopDocs;
        if (withinGroupSort.equals(Sort.RELEVANCE)) {
            mergedTopDocs = TopDocs.merge(topN, topDocs.toArray(new TopDocs[topDocs.size()]));
        } else {
            mergedTopDocs = TopDocs.merge(withinGroupSort, topN, topDocs.toArray(new TopFieldDocs[topDocs.size()]));
        }
        rb.mergedQueryCommandResults.put(query, new QueryCommandResult(mergedTopDocs, mergedMatches));
    }
    Map<Object, ShardDoc> resultIds = new HashMap<>();
    int i = 0;
    for (TopGroups<BytesRef> topGroups : rb.mergedTopGroups.values()) {
        for (GroupDocs<BytesRef> group : topGroups.groups) {
            for (ScoreDoc scoreDoc : group.scoreDocs) {
                ShardDoc solrDoc = (ShardDoc) scoreDoc;
                // Include the first if there are duplicate IDs
                if (!resultIds.containsKey(solrDoc.id)) {
                    solrDoc.positionInResponse = i++;
                    resultIds.put(solrDoc.id, solrDoc);
                }
            }
        }
    }
    for (QueryCommandResult queryCommandResult : rb.mergedQueryCommandResults.values()) {
        for (ScoreDoc scoreDoc : queryCommandResult.getTopDocs().scoreDocs) {
            ShardDoc solrDoc = (ShardDoc) scoreDoc;
            solrDoc.positionInResponse = i++;
            resultIds.put(solrDoc.id, solrDoc);
        }
    }
    rb.resultIds = resultIds;
}