Examples with ShardQueryResponse io.zulia.message.ZuliaQuery.ShardQueryResponse used on opensource projects

Example 1 with ShardQueryResponse

use of io.zulia.message.ZuliaQuery.ShardQueryResponse in project zuliasearch by zuliaio.

the class QueryCombiner method getQueryResponse.

public QueryResponse getQueryResponse() throws Exception {
    validate();
    long totalHits = 0;
    long returnedHits = 0;
    for (ShardQueryResponse sr : shardResponses) {
        totalHits += sr.getTotalHits();
        returnedHits += sr.getScoredResultList().size();
    }
    QueryResponse.Builder builder = QueryResponse.newBuilder();
    builder.setTotalHits(totalHits);
    int resultsSize = Math.min(amount, (int) returnedHits);
    Map<CountRequest, FacetCombiner> facetCombinerMap = new HashMap<>();
    Map<StatRequest, StatCombiner> statCombinerMap = new HashMap<>();
    Map<AnalysisRequest, Map<String, Term.Builder>> analysisRequestToTermMap = new HashMap<>();
    int shardIndex = 0;
    for (ShardQueryResponse sr : shardResponses) {
        for (FacetGroup fg : sr.getFacetGroupList()) {
            CountRequest countRequest = fg.getCountRequest();
            FacetCombiner facetCombiner = facetCombinerMap.computeIfAbsent(countRequest, countRequest1 -> new FacetCombiner(countRequest, shardResponses.size()));
            facetCombiner.handleFacetGroupForShard(fg, shardIndex);
        }
        for (ZuliaQuery.StatGroup sg : sr.getStatGroupList()) {
            StatRequest statRequest = sg.getStatRequest();
            StatCombiner statCombiner = statCombinerMap.computeIfAbsent(statRequest, statRequest1 -> new StatCombiner(statRequest, shardResponses.size()));
            statCombiner.handleStatGroupForShard(sg, shardIndex);
        }
        for (AnalysisResult analysisResult : sr.getAnalysisResultList()) {
            AnalysisRequest analysisRequest = analysisResult.getAnalysisRequest();
            if (!analysisRequestToTermMap.containsKey(analysisRequest)) {
                analysisRequestToTermMap.put(analysisRequest, new HashMap<>());
            }
            Map<String, Term.Builder> termMap = analysisRequestToTermMap.get(analysisRequest);
            for (Term term : analysisResult.getTermsList()) {
                String key = term.getValue();
                if (!termMap.containsKey(key)) {
                    termMap.put(key, Term.newBuilder().setValue(key).setDocFreq(0).setTermFreq(0));
                }
                Term.Builder termsBuilder = termMap.get(key);
                termsBuilder.setDocFreq(termsBuilder.getDocFreq() + term.getDocFreq());
                termsBuilder.setScore(termsBuilder.getScore() + term.getScore());
                termsBuilder.setTermFreq(termsBuilder.getTermFreq() + term.getTermFreq());
            }
        }
        shardIndex++;
    }
    for (AnalysisRequest analysisRequest : analysisRequestList) {
        Map<String, Term.Builder> termMap = analysisRequestToTermMap.get(analysisRequest);
        if (termMap != null) {
            List<Term.Builder> terms = new ArrayList<>(termMap.values());
            List<Term.Builder> topTerms = TermFreq.getTopTerms(terms, analysisRequest.getTopN(), analysisRequest.getTermSort());
            AnalysisResult.Builder analysisResultBuilder = AnalysisResult.newBuilder().setAnalysisRequest(analysisRequest);
            topTerms.forEach(analysisResultBuilder::addTerms);
            builder.addAnalysisResult(analysisResultBuilder);
        }
    }
    for (FacetCombiner facetCombiner : facetCombinerMap.values()) {
        builder.addFacetGroup(facetCombiner.getCombinedFacetGroup());
    }
    for (StatCombiner statCombiner : statCombinerMap.values()) {
        builder.addStatGroup(statCombiner.getCombinedStatGroup());
    }
    Map<String, ScoredResult[]> lastIndexResultMap = createLastIndexResultMapWithPreviousLastResults();
    List<ScoredResult> results;
    if (shardResponses.size() > 1) {
        results = mergeResults((int) returnedHits, resultsSize, lastIndexResultMap);
    } else {
        ShardQueryResponse shardQueryResponse = shardResponses.get(0);
        results = shardQueryResponse.getScoredResultList();
        if (!results.isEmpty()) {
            lastIndexResultMap.get(shardQueryResponse.getIndexName())[shardQueryResponse.getShardNumber()] = results.get(results.size() - 1);
        }
    }
    if (start == 0) {
        builder.addAllResults(results);
    } else {
        int i = 0;
        for (ScoredResult scoredResult : results) {
            if (i >= start) {
                builder.addResults(scoredResult);
            }
            i++;
        }
    }
    builder.setLastResult(createLastResult(lastIndexResultMap));
    return builder.build();
}

Example 2 with ShardQueryResponse

use of io.zulia.message.ZuliaQuery.ShardQueryResponse in project zuliasearch by zuliaio.

the class QueryCombiner method mergeResults.

private List<ScoredResult> mergeResults(int returnedHits, int resultsSize, Map<String, ScoredResult[]> lastIndexResultMap) throws Exception {
    List<ScoredResult> results = Collections.emptyList();
    boolean sorting = (sortRequest != null && !sortRequest.getFieldSortList().isEmpty());
    List<ScoredResult> mergedResults = new ArrayList<>(returnedHits);
    for (ShardQueryResponse sr : shardResponses) {
        mergedResults.addAll(sr.getScoredResultList());
    }
    if (!mergedResults.isEmpty()) {
        List<FieldSort> fieldSortList = sortRequest != null ? sortRequest.getFieldSortList() : Collections.emptyList();
        HashMap<String, FieldConfig.FieldType> sortTypeMap = createSortTypeMap(fieldSortList);
        Comparator<ScoredResult> comparator = new ZuliaPostSortingComparator(fieldSortList, sortTypeMap);
        mergedResults.sort(comparator);
        results = mergedResults.subList(0, resultsSize);
        for (ScoredResult sr : results) {
            ScoredResult[] lastForShardArr = lastIndexResultMap.get(sr.getIndexName());
            lastForShardArr[sr.getShard()] = sr;
        }
        outside: for (ZuliaIndex index : indexes) {
            String indexName = index.getIndexName();
            ScoredResult[] lastForShardArr = lastIndexResultMap.get(indexName);
            ScoredResult lastForIndex = null;
            for (ScoredResult sr : lastForShardArr) {
                if (sr != null) {
                    if (lastForIndex == null) {
                        lastForIndex = sr;
                    } else {
                        if (comparator.compare(sr, lastForIndex) > 0) {
                            lastForIndex = sr;
                        }
                    }
                }
            }
            if (lastForIndex == null) {
                // this happen when amount from index is zero
                continue;
            }
            double shardTolerance = index.getShardTolerance();
            int numberOfShards = index.getNumberOfShards();
            Map<Integer, ShardQueryResponse> shardResponseMap = indexToShardQueryResponseMap.get(indexName);
            for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) {
                ShardQueryResponse sr = shardResponseMap.get(shardNumber);
                if (sr.hasNext()) {
                    ScoredResult next = sr.getNext();
                    int compare = comparator.compare(lastForIndex, next);
                    if (compare > 0) {
                        if (sorting) {
                            String msg = "Result set did not return the most relevant sorted documents for index <" + indexName + ">\n";
                            msg += "    Last for index from shard <" + lastForIndex.getShard() + "> has sort values <" + lastForIndex.getSortValues() + ">\n";
                            msg += "    Next for shard <" + next.getShard() + ">  has sort values <" + next.getSortValues() + ">\n";
                            msg += "    Last for shards: \n";
                            msg += "      " + Arrays.toString(lastForShardArr) + "\n";
                            msg += "    Results: \n";
                            msg += "      " + results + "\n";
                            msg += "    If this happens frequently increase requestFactor or minShardRequest\n";
                            msg += "    Retrying with full request.\n";
                            log.severe(msg);
                            isShort = true;
                            break outside;
                        }
                        double diff = (Math.abs(lastForIndex.getScore() - next.getScore()));
                        if (diff > shardTolerance) {
                            String msg = "Result set did not return the most relevant documents for index <" + indexName + "> with shard tolerance <" + shardTolerance + ">\n";
                            msg += "    Last for index from shard <" + lastForIndex.getShard() + "> has score <" + lastForIndex.getScore() + ">\n";
                            msg += "    Next for shard <" + next.getShard() + "> has score <" + next.getScore() + ">\n";
                            msg += "    Last for shards: \n";
                            msg += "      " + Arrays.toString(lastForShardArr) + "\n";
                            msg += "    Results: \n";
                            msg += "      " + results + "\n";
                            msg += "    If this happens frequently increase requestFactor, minShardRequest, or shardTolerance\n";
                            msg += "    Retrying with full request.\n";
                            log.severe(msg);
                            isShort = true;
                            break outside;
                        }
                    }
                }
            }
        }
    }
    return results;
}

Example 3 with ShardQueryResponse

use of io.zulia.message.ZuliaQuery.ShardQueryResponse in project zuliasearch by zuliaio.

the class QueryCombiner method validate.

private void validate() throws Exception {
    for (InternalQueryResponse iqr : responses) {
        for (IndexShardResponse isr : iqr.getIndexShardResponseList()) {
            String indexName = isr.getIndexName();
            if (!indexToShardQueryResponseMap.containsKey(indexName)) {
                indexToShardQueryResponseMap.put(indexName, new HashMap<>());
            }
            for (ShardQueryResponse sr : isr.getShardQueryResponseList()) {
                int shardNumber = sr.getShardNumber();
                Map<Integer, ShardQueryResponse> shardResponseMap = indexToShardQueryResponseMap.get(indexName);
                if (shardResponseMap.containsKey(shardNumber)) {
                    throw new Exception("Shard <" + shardNumber + "> is repeated for <" + indexName + ">");
                } else {
                    shardResponseMap.put(shardNumber, sr);
                    shardResponses.add(sr);
                }
            }
        }
    }
    for (ZuliaIndex index : indexes) {
        int numberOfShards = index.getNumberOfShards();
        Map<Integer, ShardQueryResponse> shardResponseMap = indexToShardQueryResponseMap.get(index.getIndexName());
        if (shardResponseMap == null) {
            throw new Exception("Missing index <" + index.getIndexName() + "> in response");
        }
        if (shardResponseMap.size() != numberOfShards) {
            throw new Exception("Found <" + shardResponseMap.size() + "> expected <" + numberOfShards + ">");
        }
        for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) {
            if (!shardResponseMap.containsKey(shardNumber)) {
                throw new Exception("Missing shard <" + shardNumber + ">");
            }
        }
    }
}

Example 4 with ShardQueryResponse

use of io.zulia.message.ZuliaQuery.ShardQueryResponse in project zuliasearch by zuliaio.

the class ZuliaIndex method internalQuery.

public IndexShardResponse internalQuery(Query query, final InternalQueryRequest internalQueryRequest) throws Exception {
    QueryRequest queryRequest = internalQueryRequest.getQueryRequest();
    Set<ZuliaShard> shardsForQuery = new HashSet<>();
    for (IndexRouting indexRouting : internalQueryRequest.getIndexRoutingList()) {
        if (indexRouting.getIndex().equals(indexName)) {
            List<ZuliaShard> shardsFromRouting = getShardsFromRouting(indexRouting, queryRequest.getMasterSlaveSettings());
            shardsForQuery.addAll(shardsFromRouting);
        }
    }
    int amount = queryRequest.getAmount() + queryRequest.getStart();
    if (indexConfig.getNumberOfShards() != 1) {
        if (!queryRequest.getFetchFull() && (amount > 0)) {
            amount = (int) (((amount / numberOfShards) + indexConfig.getIndexSettings().getMinShardRequest()) * indexConfig.getIndexSettings().getRequestFactor());
        }
    }
    final int requestedAmount = amount;
    final HashMap<Integer, FieldDoc> lastScoreDocMap = new HashMap<>();
    FieldDoc after;
    ZuliaQuery.LastResult lr = queryRequest.getLastResult();
    for (ZuliaQuery.LastIndexResult lir : lr.getLastIndexResultList()) {
        if (indexName.equals(lir.getIndexName())) {
            for (ZuliaQuery.ScoredResult sr : lir.getLastForShardList()) {
                int luceneShardId = sr.getLuceneShardId();
                float score = sr.getScore();
                SortRequest sortRequest = queryRequest.getSortRequest();
                Object[] sortTerms = new Object[sortRequest.getFieldSortCount()];
                int sortTermsIndex = 0;
                ZuliaQuery.SortValues sortValues = sr.getSortValues();
                for (ZuliaQuery.FieldSort fs : sortRequest.getFieldSortList()) {
                    String sortField = fs.getSortField();
                    FieldConfig.FieldType sortType = indexConfig.getFieldTypeForSortField(sortField);
                    if (!ZuliaParser.rewriteLengthFields(sortField).equals(sortField)) {
                        sortType = FieldConfig.FieldType.NUMERIC_LONG;
                    }
                    if (sortType == null) {
                        throw new Exception(sortField + " is not defined as a sortable field");
                    }
                    ZuliaQuery.SortValue sortValue = sortValues.getSortValue(sortTermsIndex);
                    if (ZuliaConstants.SCORE_FIELD.equals(sortField)) {
                        sortTerms[sortTermsIndex] = sortValue.getFloatValue();
                    } else if (sortValue.getExists()) {
                        if (FieldTypeUtil.isNumericOrDateFieldType(sortType)) {
                            if (FieldTypeUtil.isNumericIntFieldType(sortType)) {
                                sortTerms[sortTermsIndex] = sortValue.getIntegerValue();
                            } else if (FieldTypeUtil.isNumericLongFieldType(sortType)) {
                                sortTerms[sortTermsIndex] = sortValue.getLongValue();
                            } else if (FieldTypeUtil.isNumericFloatFieldType(sortType)) {
                                sortTerms[sortTermsIndex] = sortValue.getFloatValue();
                            } else if (FieldTypeUtil.isNumericDoubleFieldType(sortType)) {
                                sortTerms[sortTermsIndex] = sortValue.getDoubleValue();
                            } else if (FieldTypeUtil.isDateFieldType(sortType)) {
                                sortTerms[sortTermsIndex] = sortValue.getDateValue();
                            } else {
                                throw new Exception("Invalid numeric sort type <" + sortType + "> for sort field <" + sortField + ">");
                            }
                        } else {
                            // string
                            sortTerms[sortTermsIndex] = new BytesRef(sortValue.getStringValue());
                        }
                    } else {
                        sortTerms[sortTermsIndex] = null;
                    }
                    sortTermsIndex++;
                }
                after = new FieldDoc(luceneShardId, score, sortTerms, sr.getShard());
                lastScoreDocMap.put(sr.getShard(), after);
            }
        }
    }
    Map<String, Similarity> fieldSimilarityMap = new HashMap<>();
    for (FieldSimilarity fieldSimilarity : queryRequest.getFieldSimilarityList()) {
        fieldSimilarityMap.put(fieldSimilarity.getField(), fieldSimilarity.getSimilarity());
    }
    for (ZuliaQuery.Query cosineSimQuery : queryRequest.getQueryList()) {
        if (cosineSimQuery.getQueryType() == ZuliaQuery.Query.QueryType.VECTOR) {
            for (String field : cosineSimQuery.getQfList()) {
                io.zulia.message.ZuliaIndex.Superbit superbitConfig = indexConfig.getSuperBitConfigForField(field);
                int sigLength = superbitConfig.getInputDim() * superbitConfig.getBatches();
                for (int i = 0; i < sigLength; i++) {
                    String fieldName = ZuliaConstants.SUPERBIT_PREFIX + "." + field + "." + i;
                    fieldSimilarityMap.put(fieldName, Similarity.CONSTANT);
                }
            }
        }
    }
    IndexShardResponse.Builder builder = IndexShardResponse.newBuilder();
    List<Future<ShardQueryResponse>> responses = new ArrayList<>();
    for (final ZuliaShard shard : shardsForQuery) {
        Future<ShardQueryResponse> response = shardPool.submit(() -> {
            QueryCacheKey queryCacheKey = null;
            if (!queryRequest.getDontCache()) {
                queryCacheKey = new QueryCacheKey(queryRequest);
            }
            return shard.queryShard(query, fieldSimilarityMap, requestedAmount, lastScoreDocMap.get(shard.getShardNumber()), queryRequest.getFacetRequest(), queryRequest.getSortRequest(), queryCacheKey, queryRequest.getResultFetchType(), queryRequest.getDocumentFieldsList(), queryRequest.getDocumentMaskedFieldsList(), queryRequest.getHighlightRequestList(), queryRequest.getAnalysisRequestList(), queryRequest.getDebug());
        });
        responses.add(response);
    }
    for (Future<ShardQueryResponse> response : responses) {
        try {
            ShardQueryResponse rs = response.get();
            builder.addShardQueryResponse(rs);
        } catch (ExecutionException e) {
            Throwable t = e.getCause();
            if (t instanceof OutOfMemoryError) {
                throw (OutOfMemoryError) t;
            }
            throw ((Exception) e.getCause());
        }
    }
    builder.setIndexName(indexName);
    return builder.build();
}