Examples with Term io.zulia.message.ZuliaBase.Term used on opensource projects

Example 1 with Term

use of io.zulia.message.ZuliaBase.Term in project zuliasearch by zuliaio.

the class TermsController method get.

@Get
@Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8", MediaType.TEXT_PLAIN + ";charset=utf-8" })
public HttpResponse<?> get(@QueryValue(ZuliaConstants.INDEX) final String indexName, @QueryValue(ZuliaConstants.FIELDS) final String field, @Nullable @QueryValue(ZuliaConstants.AMOUNT) final Integer amount, @Nullable @QueryValue(ZuliaConstants.MIN_DOC_FREQ) final Integer minDocFreq, @Nullable @QueryValue(ZuliaConstants.MIN_TERM_FREQ) final Integer minTermFreq, @Nullable @QueryValue(ZuliaConstants.START_TERM) final String startTerm, @Nullable @QueryValue(ZuliaConstants.END_TERM) final String endTerm, @Nullable @QueryValue(ZuliaConstants.TERM_FILTER) final String termFilter, @Nullable @QueryValue(ZuliaConstants.TERM_MATCH) final String termMatch, @Nullable @QueryValue(ZuliaConstants.INCLUDE_TERM) final List<String> includeTerm, @Nullable @QueryValue(ZuliaConstants.FUZZY_TERM_JSON) final String fuzzyTermJson, @QueryValue(value = ZuliaConstants.PRETTY, defaultValue = "true") Boolean pretty, @QueryValue(value = ZuliaConstants.FORMAT, defaultValue = "json") final String format) {
    ZuliaIndexManager indexManager = ZuliaNodeProvider.getZuliaNode().getIndexManager();
    GetTermsRequest.Builder termsBuilder = GetTermsRequest.newBuilder();
    termsBuilder.setIndexName(indexName);
    termsBuilder.setFieldName(field);
    if (amount != null) {
        termsBuilder.setAmount(amount);
    }
    if (minDocFreq != null) {
        termsBuilder.setMinDocFreq(minDocFreq);
    }
    if (minTermFreq != null) {
        termsBuilder.setMinTermFreq(minTermFreq);
    }
    if (startTerm != null) {
        termsBuilder.setStartTerm(startTerm);
    }
    if (endTerm != null) {
        termsBuilder.setEndTerm(endTerm);
    }
    if (termFilter != null) {
        termsBuilder.setTermFilter(termFilter);
    }
    if (termMatch != null) {
        termsBuilder.setTermMatch(termMatch);
    }
    if (includeTerm != null) {
        termsBuilder.addAllIncludeTerm(includeTerm);
    }
    if (fuzzyTermJson != null) {
        try {
            FuzzyTerm.Builder fuzzyTermBuilder = FuzzyTerm.newBuilder();
            JsonFormat.parser().merge(fuzzyTermJson, fuzzyTermBuilder);
            termsBuilder.setFuzzyTerm(fuzzyTermBuilder);
        } catch (InvalidProtocolBufferException e) {
            return HttpResponse.ok("Failed to parse analyzer json: " + e.getClass().getSimpleName() + ":" + e.getMessage()).status(ZuliaConstants.INTERNAL_ERROR);
        }
    }
    try {
        GetTermsResponse terms = indexManager.getTerms(termsBuilder.build());
        if (format.equalsIgnoreCase("json")) {
            Document document = new Document();
            document.put("index", indexName);
            document.put("field", field);
            List<Document> termsDocs = new ArrayList<>();
            for (Term term : terms.getTermList()) {
                Document termDoc = new Document();
                termDoc.put("term", term.getValue());
                termDoc.put("docFreq", term.getDocFreq());
                termDoc.put("termFreq", term.getTermFreq());
                termDoc.put("score", term.getScore());
                termsDocs.add(termDoc);
            }
            document.put("terms", termsDocs);
            String docString = document.toJson();
            if (pretty) {
                docString = JsonWriter.formatJson(docString);
            }
            return HttpResponse.ok(docString).status(ZuliaConstants.SUCCESS).contentType(MediaType.APPLICATION_JSON_TYPE + ";charset=utf-8");
        } else {
            StringBuilder csvString = new StringBuilder();
            csvString.append("term");
            csvString.append(",");
            csvString.append("termFreq");
            csvString.append(",");
            csvString.append("docFreq");
            if (termsBuilder.hasFuzzyTerm()) {
                csvString.append(",");
                csvString.append("score");
            }
            csvString.append("\n");
            for (Term term : terms.getTermList()) {
                String value = term.getValue();
                if (value.contains(",") || value.contains(" ") || value.contains("\"") || value.contains("\n")) {
                    csvString.append("\"");
                    csvString.append(value.replace("\"", "\"\""));
                    csvString.append("\"");
                } else {
                    csvString.append(value);
                }
                csvString.append(",");
                csvString.append(term.getTermFreq());
                csvString.append(",");
                csvString.append(term.getDocFreq());
                csvString.append(",");
                csvString.append(term.getScore());
                csvString.append("\n");
            }
            return HttpResponse.ok(csvString).status(ZuliaConstants.SUCCESS).contentType(MediaType.TEXT_PLAIN + ";charset=utf-8");
        }
    } catch (Exception e) {
        return HttpResponse.serverError("Failed to fetch fields for index <" + indexName + ">: " + e.getMessage()).status(ZuliaConstants.INTERNAL_ERROR);
    }
}

Example 2 with Term

use of io.zulia.message.ZuliaBase.Term in project zuliasearch by zuliaio.

the class QueryCombiner method getQueryResponse.

public QueryResponse getQueryResponse() throws Exception {
    validate();
    long totalHits = 0;
    long returnedHits = 0;
    for (ShardQueryResponse sr : shardResponses) {
        totalHits += sr.getTotalHits();
        returnedHits += sr.getScoredResultList().size();
    }
    QueryResponse.Builder builder = QueryResponse.newBuilder();
    builder.setTotalHits(totalHits);
    int resultsSize = Math.min(amount, (int) returnedHits);
    Map<CountRequest, FacetCombiner> facetCombinerMap = new HashMap<>();
    Map<StatRequest, StatCombiner> statCombinerMap = new HashMap<>();
    Map<AnalysisRequest, Map<String, Term.Builder>> analysisRequestToTermMap = new HashMap<>();
    int shardIndex = 0;
    for (ShardQueryResponse sr : shardResponses) {
        for (FacetGroup fg : sr.getFacetGroupList()) {
            CountRequest countRequest = fg.getCountRequest();
            FacetCombiner facetCombiner = facetCombinerMap.computeIfAbsent(countRequest, countRequest1 -> new FacetCombiner(countRequest, shardResponses.size()));
            facetCombiner.handleFacetGroupForShard(fg, shardIndex);
        }
        for (ZuliaQuery.StatGroup sg : sr.getStatGroupList()) {
            StatRequest statRequest = sg.getStatRequest();
            StatCombiner statCombiner = statCombinerMap.computeIfAbsent(statRequest, statRequest1 -> new StatCombiner(statRequest, shardResponses.size()));
            statCombiner.handleStatGroupForShard(sg, shardIndex);
        }
        for (AnalysisResult analysisResult : sr.getAnalysisResultList()) {
            AnalysisRequest analysisRequest = analysisResult.getAnalysisRequest();
            if (!analysisRequestToTermMap.containsKey(analysisRequest)) {
                analysisRequestToTermMap.put(analysisRequest, new HashMap<>());
            }
            Map<String, Term.Builder> termMap = analysisRequestToTermMap.get(analysisRequest);
            for (Term term : analysisResult.getTermsList()) {
                String key = term.getValue();
                if (!termMap.containsKey(key)) {
                    termMap.put(key, Term.newBuilder().setValue(key).setDocFreq(0).setTermFreq(0));
                }
                Term.Builder termsBuilder = termMap.get(key);
                termsBuilder.setDocFreq(termsBuilder.getDocFreq() + term.getDocFreq());
                termsBuilder.setScore(termsBuilder.getScore() + term.getScore());
                termsBuilder.setTermFreq(termsBuilder.getTermFreq() + term.getTermFreq());
            }
        }
        shardIndex++;
    }
    for (AnalysisRequest analysisRequest : analysisRequestList) {
        Map<String, Term.Builder> termMap = analysisRequestToTermMap.get(analysisRequest);
        if (termMap != null) {
            List<Term.Builder> terms = new ArrayList<>(termMap.values());
            List<Term.Builder> topTerms = TermFreq.getTopTerms(terms, analysisRequest.getTopN(), analysisRequest.getTermSort());
            AnalysisResult.Builder analysisResultBuilder = AnalysisResult.newBuilder().setAnalysisRequest(analysisRequest);
            topTerms.forEach(analysisResultBuilder::addTerms);
            builder.addAnalysisResult(analysisResultBuilder);
        }
    }
    for (FacetCombiner facetCombiner : facetCombinerMap.values()) {
        builder.addFacetGroup(facetCombiner.getCombinedFacetGroup());
    }
    for (StatCombiner statCombiner : statCombinerMap.values()) {
        builder.addStatGroup(statCombiner.getCombinedStatGroup());
    }
    Map<String, ScoredResult[]> lastIndexResultMap = createLastIndexResultMapWithPreviousLastResults();
    List<ScoredResult> results;
    if (shardResponses.size() > 1) {
        results = mergeResults((int) returnedHits, resultsSize, lastIndexResultMap);
    } else {
        ShardQueryResponse shardQueryResponse = shardResponses.get(0);
        results = shardQueryResponse.getScoredResultList();
        if (!results.isEmpty()) {
            lastIndexResultMap.get(shardQueryResponse.getIndexName())[shardQueryResponse.getShardNumber()] = results.get(results.size() - 1);
        }
    }
    if (start == 0) {
        builder.addAllResults(results);
    } else {
        int i = 0;
        for (ScoredResult scoredResult : results) {
            if (i >= start) {
                builder.addResults(scoredResult);
            }
            i++;
        }
    }
    builder.setLastResult(createLastResult(lastIndexResultMap));
    return builder.build();
}

Example 3 with Term

use of io.zulia.message.ZuliaBase.Term in project zuliasearch by zuliaio.

the class GetTermsRequestFederator method getResponse.

public GetTermsResponse getResponse(GetTermsRequest request) throws Exception {
    List<InternalGetTermsResponse> responses = send(request);
    TreeMap<String, Term.Builder> terms = new TreeMap<>();
    for (InternalGetTermsResponse response : responses) {
        for (GetTermsResponse gtr : response.getGetTermsResponseList()) {
            for (Term term : gtr.getTermList()) {
                String key = term.getValue();
                if (!terms.containsKey(key)) {
                    Term.Builder termBuilder = Term.newBuilder().setValue(key).setDocFreq(0).setTermFreq(0);
                    termBuilder.setScore(0);
                    terms.put(key, termBuilder);
                }
                Term.Builder builder = terms.get(key);
                builder.setDocFreq(builder.getDocFreq() + term.getDocFreq());
                builder.setTermFreq(builder.getTermFreq() + term.getTermFreq());
                builder.setScore(builder.getScore() + term.getScore());
            }
        }
    }
    GetTermsResponse.Builder responseBuilder = GetTermsResponse.newBuilder();
    Term.Builder value = null;
    int count = 0;
    int amount = request.getAmount();
    for (Term.Builder builder : terms.values()) {
        value = builder;
        if (builder.getDocFreq() >= request.getMinDocFreq() && builder.getTermFreq() >= request.getMinTermFreq()) {
            responseBuilder.addTerm(builder.build());
            count++;
        }
        if (amount != 0 && count >= amount) {
            break;
        }
    }
    if (value != null) {
        responseBuilder.setLastTerm(value.build());
    }
    return responseBuilder.build();
}