use of io.zulia.message.ZuliaServiceOuterClass.QueryResponse in project zuliasearch by zuliaio.
the class Search method execute.
@Override
public SearchResult execute(ZuliaConnection zuliaConnection) {
ZuliaServiceGrpc.ZuliaServiceBlockingStub service = zuliaConnection.getService();
QueryResponse queryResponse = service.query(getRequest());
return new SearchResult(queryResponse);
}
use of io.zulia.message.ZuliaServiceOuterClass.QueryResponse in project zuliasearch by zuliaio.
the class QueryCombiner method getQueryResponse.
public QueryResponse getQueryResponse() throws Exception {
validate();
long totalHits = 0;
long returnedHits = 0;
for (ShardQueryResponse sr : shardResponses) {
totalHits += sr.getTotalHits();
returnedHits += sr.getScoredResultList().size();
}
QueryResponse.Builder builder = QueryResponse.newBuilder();
builder.setTotalHits(totalHits);
int resultsSize = Math.min(amount, (int) returnedHits);
Map<CountRequest, FacetCombiner> facetCombinerMap = new HashMap<>();
Map<StatRequest, StatCombiner> statCombinerMap = new HashMap<>();
Map<AnalysisRequest, Map<String, Term.Builder>> analysisRequestToTermMap = new HashMap<>();
int shardIndex = 0;
for (ShardQueryResponse sr : shardResponses) {
for (FacetGroup fg : sr.getFacetGroupList()) {
CountRequest countRequest = fg.getCountRequest();
FacetCombiner facetCombiner = facetCombinerMap.computeIfAbsent(countRequest, countRequest1 -> new FacetCombiner(countRequest, shardResponses.size()));
facetCombiner.handleFacetGroupForShard(fg, shardIndex);
}
for (ZuliaQuery.StatGroup sg : sr.getStatGroupList()) {
StatRequest statRequest = sg.getStatRequest();
StatCombiner statCombiner = statCombinerMap.computeIfAbsent(statRequest, statRequest1 -> new StatCombiner(statRequest, shardResponses.size()));
statCombiner.handleStatGroupForShard(sg, shardIndex);
}
for (AnalysisResult analysisResult : sr.getAnalysisResultList()) {
AnalysisRequest analysisRequest = analysisResult.getAnalysisRequest();
if (!analysisRequestToTermMap.containsKey(analysisRequest)) {
analysisRequestToTermMap.put(analysisRequest, new HashMap<>());
}
Map<String, Term.Builder> termMap = analysisRequestToTermMap.get(analysisRequest);
for (Term term : analysisResult.getTermsList()) {
String key = term.getValue();
if (!termMap.containsKey(key)) {
termMap.put(key, Term.newBuilder().setValue(key).setDocFreq(0).setTermFreq(0));
}
Term.Builder termsBuilder = termMap.get(key);
termsBuilder.setDocFreq(termsBuilder.getDocFreq() + term.getDocFreq());
termsBuilder.setScore(termsBuilder.getScore() + term.getScore());
termsBuilder.setTermFreq(termsBuilder.getTermFreq() + term.getTermFreq());
}
}
shardIndex++;
}
for (AnalysisRequest analysisRequest : analysisRequestList) {
Map<String, Term.Builder> termMap = analysisRequestToTermMap.get(analysisRequest);
if (termMap != null) {
List<Term.Builder> terms = new ArrayList<>(termMap.values());
List<Term.Builder> topTerms = TermFreq.getTopTerms(terms, analysisRequest.getTopN(), analysisRequest.getTermSort());
AnalysisResult.Builder analysisResultBuilder = AnalysisResult.newBuilder().setAnalysisRequest(analysisRequest);
topTerms.forEach(analysisResultBuilder::addTerms);
builder.addAnalysisResult(analysisResultBuilder);
}
}
for (FacetCombiner facetCombiner : facetCombinerMap.values()) {
builder.addFacetGroup(facetCombiner.getCombinedFacetGroup());
}
for (StatCombiner statCombiner : statCombinerMap.values()) {
builder.addStatGroup(statCombiner.getCombinedStatGroup());
}
Map<String, ScoredResult[]> lastIndexResultMap = createLastIndexResultMapWithPreviousLastResults();
List<ScoredResult> results;
if (shardResponses.size() > 1) {
results = mergeResults((int) returnedHits, resultsSize, lastIndexResultMap);
} else {
ShardQueryResponse shardQueryResponse = shardResponses.get(0);
results = shardQueryResponse.getScoredResultList();
if (!results.isEmpty()) {
lastIndexResultMap.get(shardQueryResponse.getIndexName())[shardQueryResponse.getShardNumber()] = results.get(results.size() - 1);
}
}
if (start == 0) {
builder.addAllResults(results);
} else {
int i = 0;
for (ScoredResult scoredResult : results) {
if (i >= start) {
builder.addResults(scoredResult);
}
i++;
}
}
builder.setLastResult(createLastResult(lastIndexResultMap));
return builder.build();
}
use of io.zulia.message.ZuliaServiceOuterClass.QueryResponse in project zuliasearch by zuliaio.
the class Query method execute.
@Override
public SearchResult execute(ZuliaConnection zuliaConnection) {
ZuliaServiceGrpc.ZuliaServiceBlockingStub service = zuliaConnection.getService();
QueryResponse queryResponse = service.query(getRequest());
return new SearchResult(queryResponse);
}
use of io.zulia.message.ZuliaServiceOuterClass.QueryResponse in project zuliasearch by zuliaio.
the class QueryController method get.
@Get
@Produces({ MediaType.APPLICATION_JSON + ";charset=utf-8", MediaType.TEXT_PLAIN + ";charset=utf-8" })
public HttpResponse<?> get(@QueryValue(ZuliaConstants.INDEX) List<String> indexName, @QueryValue(value = ZuliaConstants.QUERY, defaultValue = "*:*") String query, @Nullable @QueryValue(ZuliaConstants.QUERY_FIELD) List<String> queryFields, @Nullable @QueryValue(ZuliaConstants.FILTER_QUERY) List<String> filterQueries, @Nullable @QueryValue(ZuliaConstants.QUERY_JSON) List<String> queryJsonList, @Nullable @QueryValue(ZuliaConstants.FIELDS) List<String> fields, @QueryValue(value = ZuliaConstants.FETCH, defaultValue = "true") Boolean fetch, @QueryValue(value = ZuliaConstants.ROWS, defaultValue = "0") Integer rows, @Nullable @QueryValue(ZuliaConstants.FACET) List<String> facet, @Nullable @QueryValue(ZuliaConstants.DRILL_DOWN) List<String> drillDowns, @Nullable @QueryValue(ZuliaConstants.DEFAULT_OP) String defaultOperator, @Nullable @QueryValue(ZuliaConstants.SORT) List<String> sort, @QueryValue(value = ZuliaConstants.PRETTY, defaultValue = "true") Boolean pretty, @Nullable @QueryValue(value = ZuliaConstants.DISMAX, defaultValue = "false") Boolean dismax, @Nullable @QueryValue(ZuliaConstants.DISMAX_TIE) Float dismaxTie, @Nullable @QueryValue(ZuliaConstants.MIN_MATCH) Integer mm, @Nullable @QueryValue(ZuliaConstants.SIMILARITY) List<String> similarity, @QueryValue(value = ZuliaConstants.DEBUG, defaultValue = "false") Boolean debug, @QueryValue(value = ZuliaConstants.DONT_CACHE, defaultValue = "true") Boolean dontCache, @Nullable @QueryValue(ZuliaConstants.START) Integer start, @Nullable @QueryValue(ZuliaConstants.HIGHLIGHT) List<String> highlightList, @Nullable @QueryValue(ZuliaConstants.HIGHLIGHT_JSON) List<String> highlightJsonList, @Nullable @QueryValue(ZuliaConstants.ANALYZE_JSON) List<String> analyzeJsonList, @QueryValue(value = ZuliaConstants.FORMAT, defaultValue = "json") String format, @QueryValue(value = ZuliaConstants.BATCH, defaultValue = "false") Boolean batch, @QueryValue(value = ZuliaConstants.BATCH_SIZE, defaultValue = "500") Integer batchSize, @Nullable @QueryValue(ZuliaConstants.CURSOR) String cursor, @QueryValue(value = ZuliaConstants.TRUNCATE, defaultValue = "false") Boolean truncate) {
ZuliaIndexManager indexManager = ZuliaNodeProvider.getZuliaNode().getIndexManager();
QueryRequest.Builder qrBuilder = QueryRequest.newBuilder().addAllIndex(indexName);
try {
boolean outputCursor = false;
if (cursor != null) {
if (!cursor.equals("0")) {
qrBuilder.setLastResult(CursorHelper.getLastResultFromCursor(cursor));
}
outputCursor = true;
if (sort == null || sort.isEmpty()) {
return HttpResponse.created("Sort on unique value or value combination is required to use a cursor (i.e. id or title,id)").status(ZuliaConstants.INTERNAL_ERROR);
}
}
if (debug != null) {
qrBuilder.setDebug(debug);
}
if (start != null) {
qrBuilder.setStart(start);
}
if (dontCache != null) {
qrBuilder.setDontCache(dontCache);
}
Query.Builder mainQueryBuilder = Query.newBuilder();
if (query != null) {
mainQueryBuilder.setQ(query);
}
if (mm != null) {
mainQueryBuilder.setMm(mm);
}
if (dismax != null) {
mainQueryBuilder.setDismax(dismax);
if (dismaxTie != null) {
mainQueryBuilder.setDismaxTie(dismaxTie);
}
}
if (queryFields != null) {
mainQueryBuilder.addAllQf(queryFields);
}
if (defaultOperator != null) {
if (defaultOperator.equalsIgnoreCase("AND")) {
mainQueryBuilder.setDefaultOp(Query.Operator.AND);
} else if (defaultOperator.equalsIgnoreCase("OR")) {
mainQueryBuilder.setDefaultOp(Query.Operator.OR);
} else {
HttpResponse.created("Invalid default operator <" + defaultOperator + ">").status(ZuliaConstants.INTERNAL_ERROR);
}
}
mainQueryBuilder.setQueryType(Query.QueryType.SCORE_MUST);
qrBuilder.addQuery(mainQueryBuilder);
if (similarity != null) {
for (String sim : similarity) {
if (sim.contains(":")) {
int i = sim.indexOf(":");
String field = sim.substring(0, i);
String simType = sim.substring(i + 1);
FieldSimilarity.Builder fieldSimilarity = FieldSimilarity.newBuilder();
fieldSimilarity.setField(field);
if (simType.equalsIgnoreCase("bm25")) {
fieldSimilarity.setSimilarity(Similarity.BM25);
} else if (simType.equalsIgnoreCase("constant")) {
fieldSimilarity.setSimilarity(Similarity.CONSTANT);
} else if (simType.equalsIgnoreCase("tf")) {
fieldSimilarity.setSimilarity(Similarity.TF);
} else if (simType.equalsIgnoreCase("tfidf")) {
fieldSimilarity.setSimilarity(Similarity.TFIDF);
} else {
HttpResponse.created("Unknown similarity type <" + simType + ">").status(ZuliaConstants.INTERNAL_ERROR);
}
qrBuilder.addFieldSimilarity(fieldSimilarity);
} else {
HttpResponse.created("Similarity <" + sim + "> should be in the form field:simType").status(ZuliaConstants.INTERNAL_ERROR);
}
}
}
if (filterQueries != null) {
for (String filterQuery : filterQueries) {
Query filterQueryBuilder = Query.newBuilder().setQ(filterQuery).setQueryType(Query.QueryType.FILTER).build();
qrBuilder.addQuery(filterQueryBuilder);
}
}
if (queryJsonList != null) {
for (String queryJson : queryJsonList) {
try {
Query.Builder subQueryBuilder = Query.newBuilder();
JsonFormat.parser().merge(queryJson, subQueryBuilder);
qrBuilder.addQuery(subQueryBuilder);
} catch (InvalidProtocolBufferException e) {
return HttpResponse.created("Failed to parse query json: " + e.getClass().getSimpleName() + ":" + e.getMessage()).status(ZuliaConstants.INTERNAL_ERROR);
}
}
}
if (highlightList != null) {
for (String hl : highlightList) {
HighlightRequest highlightRequest = HighlightRequest.newBuilder().setField(hl).build();
qrBuilder.addHighlightRequest(highlightRequest);
}
}
if (highlightJsonList != null) {
for (String hlJson : highlightJsonList) {
try {
HighlightRequest.Builder hlBuilder = HighlightRequest.newBuilder();
JsonFormat.parser().merge(hlJson, hlBuilder);
qrBuilder.addHighlightRequest(hlBuilder);
} catch (InvalidProtocolBufferException e) {
return HttpResponse.created("Failed to parse highlight json: " + e.getClass().getSimpleName() + ":" + e.getMessage()).status(ZuliaConstants.INTERNAL_ERROR);
}
}
}
if (analyzeJsonList != null) {
for (String alJson : analyzeJsonList) {
try {
AnalysisRequest.Builder analyzeRequestBuilder = AnalysisRequest.newBuilder();
JsonFormat.parser().merge(alJson, analyzeRequestBuilder);
qrBuilder.addAnalysisRequest(analyzeRequestBuilder);
} catch (InvalidProtocolBufferException e) {
return HttpResponse.created("Failed to parse analyzer json: " + e.getClass().getSimpleName() + ":" + e.getMessage()).status(ZuliaConstants.INTERNAL_ERROR);
}
}
}
if (fields != null) {
for (String field : fields) {
if (field.startsWith("-")) {
qrBuilder.addDocumentMaskedFields(field.substring(1));
} else {
qrBuilder.addDocumentFields(field);
}
}
}
qrBuilder.setResultFetchType(FetchType.FULL);
if (fetch != null && !fetch) {
qrBuilder.setResultFetchType(FetchType.NONE);
}
FacetRequest.Builder frBuilder = FacetRequest.newBuilder();
if (facet != null) {
for (String f : facet) {
Integer count = null;
if (f.contains(":")) {
String countString = f.substring(f.indexOf(":") + 1);
f = f.substring(0, f.indexOf(":"));
try {
count = Integer.parseInt(countString);
} catch (Exception e) {
return HttpResponse.created("Invalid facet count <" + countString + "> for facet <" + f + ">").status(ZuliaConstants.INTERNAL_ERROR);
}
}
CountRequest.Builder countBuilder = CountRequest.newBuilder();
Facet zuliaFacet = Facet.newBuilder().setLabel(f).build();
CountRequest.Builder facetBuilder = countBuilder.setFacetField(zuliaFacet);
if (count != null) {
facetBuilder.setMaxFacets(count);
}
frBuilder.addCountRequest(facetBuilder);
}
}
if (drillDowns != null) {
for (String drillDown : drillDowns) {
if (drillDown.contains(":")) {
String value = drillDown.substring(drillDown.indexOf(":") + 1);
String field = drillDown.substring(0, drillDown.indexOf(":"));
frBuilder.addDrillDown(Facet.newBuilder().setLabel(field).setValue(value));
}
}
}
qrBuilder.setFacetRequest(frBuilder);
if (sort != null) {
SortRequest.Builder sortRequest = SortRequest.newBuilder();
for (String sortField : sort) {
FieldSort.Builder fieldSort = FieldSort.newBuilder();
if (sortField.contains(":")) {
String sortDir = sortField.substring(sortField.indexOf(":") + 1);
sortField = sortField.substring(0, sortField.indexOf(":"));
if ("-1".equals(sortDir) || "DESC".equalsIgnoreCase(sortDir)) {
fieldSort.setDirection(FieldSort.Direction.DESCENDING);
} else if ("1".equals(sortDir) || "ASC".equalsIgnoreCase(sortDir)) {
fieldSort.setDirection(FieldSort.Direction.ASCENDING);
} else {
return HttpResponse.created("Invalid sort direction <" + sortDir + "> for field <" + sortField + ">. Expecting -1/1 or DESC/ASC").status(ZuliaConstants.INTERNAL_ERROR);
}
}
fieldSort.setSortField(sortField);
sortRequest.addFieldSort(fieldSort);
}
qrBuilder.setSortRequest(sortRequest);
}
qrBuilder.setAmount(rows);
if (format.equals("json")) {
QueryResponse qr = indexManager.query(qrBuilder.build());
String response = getStandardResponse(qr, !pretty, outputCursor, truncate);
if (pretty) {
response = JsonWriter.formatJson(response);
}
return HttpResponse.ok(response).status(ZuliaConstants.SUCCESS).contentType(MediaType.APPLICATION_JSON_TYPE);
} else {
if (fields != null && !fields.isEmpty()) {
if (batch) {
qrBuilder.setAmount(batchSize);
Writable writable = output -> {
try {
QueryResponse qr = indexManager.query(qrBuilder.build());
String header = buildHeaderForCSV(fields);
output.write(header);
output.flush();
int count = 0;
while (qr.getResultsList().size() > 0) {
for (ScoredResult scoredResult : qr.getResultsList()) {
Document doc = ResultHelper.getDocumentFromScoredResult(scoredResult);
appendDocument(fields, null, output, doc);
count++;
if (count % 1000 == 0) {
LOG.info("Docs processed so far: " + count);
}
}
qrBuilder.setLastResult(qr.getLastResult());
qr = indexManager.query(qrBuilder.build());
}
} catch (Exception e) {
e.printStackTrace();
}
};
LocalDateTime now = LocalDateTime.now();
DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyy-MM-dd-H-mm-ss");
return HttpResponse.ok(writable).status(ZuliaConstants.SUCCESS).header("content-disposition", "attachment; filename = " + "zuliaDownload_" + now.format(formatter) + ".csv").contentType(MediaType.APPLICATION_OCTET_STREAM);
} else {
QueryResponse qr = indexManager.query(qrBuilder.build());
String response = getCSVDocumentResponse(fields, qr);
return HttpResponse.ok(response).status(ZuliaConstants.SUCCESS).contentType(MediaType.TEXT_PLAIN + ";charset=utf-8");
}
} else if (facet != null && !facet.isEmpty() && rows == 0) {
QueryResponse qr = indexManager.query(qrBuilder.build());
StringBuilder response = new StringBuilder();
response.append("facetName,facetKey,facetValue\n");
for (FacetGroup facetGroup : qr.getFacetGroupList()) {
for (FacetCount facetCount : facetGroup.getFacetCountList()) {
response.append(facetGroup.getCountRequest().getFacetField().getLabel());
response.append(",");
response.append("\"").append(facetCount.getFacet()).append("\"");
response.append(",");
response.append(Long.valueOf(facetCount.getCount()));
response.append("\n");
}
}
return HttpResponse.ok(response.toString()).status(ZuliaConstants.SUCCESS).contentType(MediaType.TEXT_PLAIN + ";charset=utf-8");
} else {
return HttpResponse.ok("Please specify fields to be exported i.e. fl=title&fl=abstract or the facets to be exported i.e. facet=issn&facet=pubYear&rows=0").status(ZuliaConstants.SUCCESS).contentType(MediaType.TEXT_PLAIN + ";charset=utf-8");
}
}
} catch (Exception e) {
LOG.log(Level.SEVERE, e.getMessage(), e);
return HttpResponse.serverError(e.getClass().getSimpleName() + ":" + e.getMessage()).status(ZuliaConstants.INTERNAL_ERROR);
}
}
use of io.zulia.message.ZuliaServiceOuterClass.QueryResponse in project zuliasearch by zuliaio.
the class QueryRequestFederator method getResponse.
public QueryResponse getResponse(QueryRequest request) throws Exception {
long queryId = queryNumber.getAndIncrement();
long start = System.currentTimeMillis();
String queryJson = JsonFormat.printer().print(request);
LOG.info("Running id <" + queryId + "> query <" + queryJson + ">");
List<InternalQueryResponse> results = send(request);
QueryCombiner queryCombiner = new QueryCombiner(indexes, request, results);
QueryResponse qr = queryCombiner.getQueryResponse();
if (!queryCombiner.isShort()) {
long end = System.currentTimeMillis();
LOG.info("Finished query id <" + queryId + "> in " + (end - start) + "ms");
return qr;
} else {
if (!request.getFetchFull()) {
QueryRequest newRequest = request.toBuilder().setFetchFull(true).build();
long end = System.currentTimeMillis();
LOG.info("Finished query id <" + queryId + "> in " + (end - start) + "ms");
return getResponse(newRequest);
}
throw new Exception("Full fetch request is short");
}
}
Aggregations