Examples with FacetGroup io.zulia.message.ZuliaQuery.FacetGroup used on opensource projects

Example 1 with FacetGroup

use of io.zulia.message.ZuliaQuery.FacetGroup in project zuliasearch by zuliaio.

the class FacetCombiner method getCombinedFacetGroup.

public FacetGroup getCombinedFacetGroup() {
    if (facetGroups.size() == 1) {
        return facetGroups.get(0).getFacetGroup();
    } else {
        Map<String, AtomicLong> facetCounts = new HashMap<>();
        Map<String, FixedBitSet> shardsReturned = new HashMap<>();
        FixedBitSet fullResults = new FixedBitSet(shardReponses);
        long[] minForShard = new long[shardReponses];
        for (FacetGroupWithShardIndex facetGroupWithShardIndex : facetGroups) {
            FacetGroup fg = facetGroupWithShardIndex.getFacetGroup();
            int shardIndex = facetGroupWithShardIndex.getShardIndex();
            for (FacetCount fc : fg.getFacetCountList()) {
                String facet = fc.getFacet();
                AtomicLong facetSum = facetCounts.get(facet);
                FixedBitSet shardSet = shardsReturned.get(facet);
                if (facetSum == null) {
                    facetSum = new AtomicLong();
                    facetCounts.put(facet, facetSum);
                    shardSet = new FixedBitSet(shardReponses);
                    shardsReturned.put(facet, shardSet);
                }
                long count = fc.getCount();
                facetSum.addAndGet(count);
                shardSet.set(shardIndex);
                minForShard[shardIndex] = count;
            }
            int shardFacets = countRequest.getShardFacets();
            int facetCountCount = fg.getFacetCountCount();
            if (facetCountCount < shardFacets || (shardFacets == -1)) {
                fullResults.set(shardIndex);
                minForShard[shardIndex] = 0;
            }
        }
        FacetGroup.Builder fg = FacetGroup.newBuilder();
        fg.setCountRequest(countRequest);
        int numberOfShards = shardIndexes.length;
        long maxValuePossibleMissing = 0;
        for (int i = 0; i < numberOfShards; i++) {
            maxValuePossibleMissing += minForShard[i];
        }
        boolean computeError = countRequest.getMaxFacets() > 0 && countRequest.getShardFacets() > 0 && numberOfShards > 1;
        boolean computePossibleMissing = computeError && (maxValuePossibleMissing != 0);
        SortedSet<FacetCountResult> sortedFacetResults = facetCounts.keySet().stream().map(facet -> new FacetCountResult(facet, facetCounts.get(facet).get())).collect(Collectors.toCollection(TreeSet::new));
        int maxCount = countRequest.getMaxFacets();
        long minCountReturned = 0;
        int count = 0;
        for (FacetCountResult facet : sortedFacetResults) {
            FixedBitSet shardCount = shardsReturned.get(facet.getFacet());
            shardCount.or(fullResults);
            FacetCount.Builder facetCountBuilder = FacetCount.newBuilder().setFacet(facet.getFacet()).setCount(facet.getCount());
            long maxWithError = 0;
            if (computeError) {
                long maxError = 0;
                if (shardCount.cardinality() < numberOfShards) {
                    for (int i = 0; i < numberOfShards; i++) {
                        if (!shardCount.get(i)) {
                            maxError += minForShard[i];
                        }
                    }
                }
                facetCountBuilder.setMaxError(maxError);
                maxWithError = maxError + facet.getCount();
            }
            count++;
            if (maxCount > 0 && count > maxCount) {
                if (computePossibleMissing) {
                    if (maxWithError > maxValuePossibleMissing) {
                        maxValuePossibleMissing = maxWithError;
                    }
                } else {
                    break;
                }
            } else {
                fg.addFacetCount(facetCountBuilder);
                minCountReturned = facet.getCount();
            }
        }
        if (!sortedFacetResults.isEmpty()) {
            if (maxValuePossibleMissing > minCountReturned) {
                fg.setPossibleMissing(true);
                fg.setMaxValuePossibleMissing(maxValuePossibleMissing);
            }
        }
        return fg.build();
    }
}

Example 2 with FacetGroup

use of io.zulia.message.ZuliaQuery.FacetGroup in project zuliasearch by zuliaio.

the class QueryCombiner method getQueryResponse.

public QueryResponse getQueryResponse() throws Exception {
    validate();
    long totalHits = 0;
    long returnedHits = 0;
    for (ShardQueryResponse sr : shardResponses) {
        totalHits += sr.getTotalHits();
        returnedHits += sr.getScoredResultList().size();
    }
    QueryResponse.Builder builder = QueryResponse.newBuilder();
    builder.setTotalHits(totalHits);
    int resultsSize = Math.min(amount, (int) returnedHits);
    Map<CountRequest, FacetCombiner> facetCombinerMap = new HashMap<>();
    Map<StatRequest, StatCombiner> statCombinerMap = new HashMap<>();
    Map<AnalysisRequest, Map<String, Term.Builder>> analysisRequestToTermMap = new HashMap<>();
    int shardIndex = 0;
    for (ShardQueryResponse sr : shardResponses) {
        for (FacetGroup fg : sr.getFacetGroupList()) {
            CountRequest countRequest = fg.getCountRequest();
            FacetCombiner facetCombiner = facetCombinerMap.computeIfAbsent(countRequest, countRequest1 -> new FacetCombiner(countRequest, shardResponses.size()));
            facetCombiner.handleFacetGroupForShard(fg, shardIndex);
        }
        for (ZuliaQuery.StatGroup sg : sr.getStatGroupList()) {
            StatRequest statRequest = sg.getStatRequest();
            StatCombiner statCombiner = statCombinerMap.computeIfAbsent(statRequest, statRequest1 -> new StatCombiner(statRequest, shardResponses.size()));
            statCombiner.handleStatGroupForShard(sg, shardIndex);
        }
        for (AnalysisResult analysisResult : sr.getAnalysisResultList()) {
            AnalysisRequest analysisRequest = analysisResult.getAnalysisRequest();
            if (!analysisRequestToTermMap.containsKey(analysisRequest)) {
                analysisRequestToTermMap.put(analysisRequest, new HashMap<>());
            }
            Map<String, Term.Builder> termMap = analysisRequestToTermMap.get(analysisRequest);
            for (Term term : analysisResult.getTermsList()) {
                String key = term.getValue();
                if (!termMap.containsKey(key)) {
                    termMap.put(key, Term.newBuilder().setValue(key).setDocFreq(0).setTermFreq(0));
                }
                Term.Builder termsBuilder = termMap.get(key);
                termsBuilder.setDocFreq(termsBuilder.getDocFreq() + term.getDocFreq());
                termsBuilder.setScore(termsBuilder.getScore() + term.getScore());
                termsBuilder.setTermFreq(termsBuilder.getTermFreq() + term.getTermFreq());
            }
        }
        shardIndex++;
    }
    for (AnalysisRequest analysisRequest : analysisRequestList) {
        Map<String, Term.Builder> termMap = analysisRequestToTermMap.get(analysisRequest);
        if (termMap != null) {
            List<Term.Builder> terms = new ArrayList<>(termMap.values());
            List<Term.Builder> topTerms = TermFreq.getTopTerms(terms, analysisRequest.getTopN(), analysisRequest.getTermSort());
            AnalysisResult.Builder analysisResultBuilder = AnalysisResult.newBuilder().setAnalysisRequest(analysisRequest);
            topTerms.forEach(analysisResultBuilder::addTerms);
            builder.addAnalysisResult(analysisResultBuilder);
        }
    }
    for (FacetCombiner facetCombiner : facetCombinerMap.values()) {
        builder.addFacetGroup(facetCombiner.getCombinedFacetGroup());
    }
    for (StatCombiner statCombiner : statCombinerMap.values()) {
        builder.addStatGroup(statCombiner.getCombinedStatGroup());
    }
    Map<String, ScoredResult[]> lastIndexResultMap = createLastIndexResultMapWithPreviousLastResults();
    List<ScoredResult> results;
    if (shardResponses.size() > 1) {
        results = mergeResults((int) returnedHits, resultsSize, lastIndexResultMap);
    } else {
        ShardQueryResponse shardQueryResponse = shardResponses.get(0);
        results = shardQueryResponse.getScoredResultList();
        if (!results.isEmpty()) {
            lastIndexResultMap.get(shardQueryResponse.getIndexName())[shardQueryResponse.getShardNumber()] = results.get(results.size() - 1);
        }
    }
    if (start == 0) {
        builder.addAllResults(results);
    } else {
        int i = 0;
        for (ScoredResult scoredResult : results) {
            if (i >= start) {
                builder.addResults(scoredResult);
            }
            i++;
        }
    }
    builder.setLastResult(createLastResult(lastIndexResultMap));
    return builder.build();
}