use of io.zulia.message.ZuliaQuery.ScoredResult in project zuliasearch by zuliaio.
the class QueryCombiner method getQueryResponse.
public QueryResponse getQueryResponse() throws Exception {
validate();
long totalHits = 0;
long returnedHits = 0;
for (ShardQueryResponse sr : shardResponses) {
totalHits += sr.getTotalHits();
returnedHits += sr.getScoredResultList().size();
}
QueryResponse.Builder builder = QueryResponse.newBuilder();
builder.setTotalHits(totalHits);
int resultsSize = Math.min(amount, (int) returnedHits);
Map<CountRequest, FacetCombiner> facetCombinerMap = new HashMap<>();
Map<StatRequest, StatCombiner> statCombinerMap = new HashMap<>();
Map<AnalysisRequest, Map<String, Term.Builder>> analysisRequestToTermMap = new HashMap<>();
int shardIndex = 0;
for (ShardQueryResponse sr : shardResponses) {
for (FacetGroup fg : sr.getFacetGroupList()) {
CountRequest countRequest = fg.getCountRequest();
FacetCombiner facetCombiner = facetCombinerMap.computeIfAbsent(countRequest, countRequest1 -> new FacetCombiner(countRequest, shardResponses.size()));
facetCombiner.handleFacetGroupForShard(fg, shardIndex);
}
for (ZuliaQuery.StatGroup sg : sr.getStatGroupList()) {
StatRequest statRequest = sg.getStatRequest();
StatCombiner statCombiner = statCombinerMap.computeIfAbsent(statRequest, statRequest1 -> new StatCombiner(statRequest, shardResponses.size()));
statCombiner.handleStatGroupForShard(sg, shardIndex);
}
for (AnalysisResult analysisResult : sr.getAnalysisResultList()) {
AnalysisRequest analysisRequest = analysisResult.getAnalysisRequest();
if (!analysisRequestToTermMap.containsKey(analysisRequest)) {
analysisRequestToTermMap.put(analysisRequest, new HashMap<>());
}
Map<String, Term.Builder> termMap = analysisRequestToTermMap.get(analysisRequest);
for (Term term : analysisResult.getTermsList()) {
String key = term.getValue();
if (!termMap.containsKey(key)) {
termMap.put(key, Term.newBuilder().setValue(key).setDocFreq(0).setTermFreq(0));
}
Term.Builder termsBuilder = termMap.get(key);
termsBuilder.setDocFreq(termsBuilder.getDocFreq() + term.getDocFreq());
termsBuilder.setScore(termsBuilder.getScore() + term.getScore());
termsBuilder.setTermFreq(termsBuilder.getTermFreq() + term.getTermFreq());
}
}
shardIndex++;
}
for (AnalysisRequest analysisRequest : analysisRequestList) {
Map<String, Term.Builder> termMap = analysisRequestToTermMap.get(analysisRequest);
if (termMap != null) {
List<Term.Builder> terms = new ArrayList<>(termMap.values());
List<Term.Builder> topTerms = TermFreq.getTopTerms(terms, analysisRequest.getTopN(), analysisRequest.getTermSort());
AnalysisResult.Builder analysisResultBuilder = AnalysisResult.newBuilder().setAnalysisRequest(analysisRequest);
topTerms.forEach(analysisResultBuilder::addTerms);
builder.addAnalysisResult(analysisResultBuilder);
}
}
for (FacetCombiner facetCombiner : facetCombinerMap.values()) {
builder.addFacetGroup(facetCombiner.getCombinedFacetGroup());
}
for (StatCombiner statCombiner : statCombinerMap.values()) {
builder.addStatGroup(statCombiner.getCombinedStatGroup());
}
Map<String, ScoredResult[]> lastIndexResultMap = createLastIndexResultMapWithPreviousLastResults();
List<ScoredResult> results;
if (shardResponses.size() > 1) {
results = mergeResults((int) returnedHits, resultsSize, lastIndexResultMap);
} else {
ShardQueryResponse shardQueryResponse = shardResponses.get(0);
results = shardQueryResponse.getScoredResultList();
if (!results.isEmpty()) {
lastIndexResultMap.get(shardQueryResponse.getIndexName())[shardQueryResponse.getShardNumber()] = results.get(results.size() - 1);
}
}
if (start == 0) {
builder.addAllResults(results);
} else {
int i = 0;
for (ScoredResult scoredResult : results) {
if (i >= start) {
builder.addResults(scoredResult);
}
i++;
}
}
builder.setLastResult(createLastResult(lastIndexResultMap));
return builder.build();
}
use of io.zulia.message.ZuliaQuery.ScoredResult in project zuliasearch by zuliaio.
the class CursorHelper method getUniqueSortedCursor.
public static String getUniqueSortedCursor(LastResult lastResult) {
LastResult.Builder lastResultBuilder = LastResult.newBuilder();
for (LastIndexResult lastIndexResult : lastResult.getLastIndexResultList()) {
LastIndexResult.Builder lastIndexResultBuilder = LastIndexResult.newBuilder();
lastIndexResultBuilder.setIndexName(lastIndexResult.getIndexName());
for (ScoredResult scoredResult : lastIndexResult.getLastForShardList()) {
ScoredResult.Builder scoredResultBuilder = ScoredResult.newBuilder(scoredResult).clearScore().clearResultDocument();
lastIndexResultBuilder.addLastForShard(scoredResultBuilder);
}
lastResultBuilder.addLastIndexResult(lastIndexResultBuilder);
}
return new String(Base64.getEncoder().encode(lastResultBuilder.build().toByteArray()), StandardCharsets.UTF_8);
}
use of io.zulia.message.ZuliaQuery.ScoredResult in project zuliasearch by zuliaio.
the class BatchDelete method deleteDocumentFromQueryResult.
public BatchDelete deleteDocumentFromQueryResult(SearchResult queryResult) {
for (ScoredResult sr : queryResult.getResults()) {
Delete delete = new DeleteDocument(sr.getUniqueId(), sr.getIndexName());
deletes.add(delete);
}
return this;
}
use of io.zulia.message.ZuliaQuery.ScoredResult in project zuliasearch by zuliaio.
the class BatchFetch method addFetchDocumentsFromResults.
public BatchFetch addFetchDocumentsFromResults(Collection<ZuliaQuery.ScoredResult> scoredResults) {
for (ScoredResult scoredResult : scoredResults) {
Fetch f = new Fetch(scoredResult.getUniqueId(), scoredResult.getIndexName());
f.setResultFetchType(FetchType.FULL);
f.setAssociatedFetchType(FetchType.NONE);
fetchList.add(f);
}
return this;
}
use of io.zulia.message.ZuliaQuery.ScoredResult in project zuliasearch by zuliaio.
the class QueryCombiner method mergeResults.
private List<ScoredResult> mergeResults(int returnedHits, int resultsSize, Map<String, ScoredResult[]> lastIndexResultMap) throws Exception {
List<ScoredResult> results = Collections.emptyList();
boolean sorting = (sortRequest != null && !sortRequest.getFieldSortList().isEmpty());
List<ScoredResult> mergedResults = new ArrayList<>(returnedHits);
for (ShardQueryResponse sr : shardResponses) {
mergedResults.addAll(sr.getScoredResultList());
}
if (!mergedResults.isEmpty()) {
List<FieldSort> fieldSortList = sortRequest != null ? sortRequest.getFieldSortList() : Collections.emptyList();
HashMap<String, FieldConfig.FieldType> sortTypeMap = createSortTypeMap(fieldSortList);
Comparator<ScoredResult> comparator = new ZuliaPostSortingComparator(fieldSortList, sortTypeMap);
mergedResults.sort(comparator);
results = mergedResults.subList(0, resultsSize);
for (ScoredResult sr : results) {
ScoredResult[] lastForShardArr = lastIndexResultMap.get(sr.getIndexName());
lastForShardArr[sr.getShard()] = sr;
}
outside: for (ZuliaIndex index : indexes) {
String indexName = index.getIndexName();
ScoredResult[] lastForShardArr = lastIndexResultMap.get(indexName);
ScoredResult lastForIndex = null;
for (ScoredResult sr : lastForShardArr) {
if (sr != null) {
if (lastForIndex == null) {
lastForIndex = sr;
} else {
if (comparator.compare(sr, lastForIndex) > 0) {
lastForIndex = sr;
}
}
}
}
if (lastForIndex == null) {
// this happen when amount from index is zero
continue;
}
double shardTolerance = index.getShardTolerance();
int numberOfShards = index.getNumberOfShards();
Map<Integer, ShardQueryResponse> shardResponseMap = indexToShardQueryResponseMap.get(indexName);
for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) {
ShardQueryResponse sr = shardResponseMap.get(shardNumber);
if (sr.hasNext()) {
ScoredResult next = sr.getNext();
int compare = comparator.compare(lastForIndex, next);
if (compare > 0) {
if (sorting) {
String msg = "Result set did not return the most relevant sorted documents for index <" + indexName + ">\n";
msg += " Last for index from shard <" + lastForIndex.getShard() + "> has sort values <" + lastForIndex.getSortValues() + ">\n";
msg += " Next for shard <" + next.getShard() + "> has sort values <" + next.getSortValues() + ">\n";
msg += " Last for shards: \n";
msg += " " + Arrays.toString(lastForShardArr) + "\n";
msg += " Results: \n";
msg += " " + results + "\n";
msg += " If this happens frequently increase requestFactor or minShardRequest\n";
msg += " Retrying with full request.\n";
log.severe(msg);
isShort = true;
break outside;
}
double diff = (Math.abs(lastForIndex.getScore() - next.getScore()));
if (diff > shardTolerance) {
String msg = "Result set did not return the most relevant documents for index <" + indexName + "> with shard tolerance <" + shardTolerance + ">\n";
msg += " Last for index from shard <" + lastForIndex.getShard() + "> has score <" + lastForIndex.getScore() + ">\n";
msg += " Next for shard <" + next.getShard() + "> has score <" + next.getScore() + ">\n";
msg += " Last for shards: \n";
msg += " " + Arrays.toString(lastForShardArr) + "\n";
msg += " Results: \n";
msg += " " + results + "\n";
msg += " If this happens frequently increase requestFactor, minShardRequest, or shardTolerance\n";
msg += " Retrying with full request.\n";
log.severe(msg);
isShort = true;
break outside;
}
}
}
}
}
}
return results;
}
Aggregations