use of io.zulia.message.ZuliaQuery.FieldSort in project zuliasearch by zuliaio.
the class QueryCombiner method createSortTypeMap.
private HashMap<String, FieldConfig.FieldType> createSortTypeMap(List<FieldSort> fieldSortList) throws Exception {
HashMap<String, FieldConfig.FieldType> sortTypeMap = new HashMap<>();
if (!fieldSortList.isEmpty()) {
for (FieldSort fieldSort : fieldSortList) {
String sortField = fieldSort.getSortField();
if (ZuliaParser.rewriteLengthFields(sortField).equals(sortField)) {
for (ZuliaIndex index : indexes) {
FieldConfig.FieldType currentSortType = sortTypeMap.get(sortField);
FieldConfig.FieldType indexSortType = index.getSortFieldType(sortField);
if (currentSortType == null) {
sortTypeMap.put(sortField, indexSortType);
} else {
if (!currentSortType.equals(indexSortType)) {
log.severe("Sort fields must be defined the same in all indexes searched in a single query");
String message = "Cannot sort on field <" + sortField + ">: found type: <" + currentSortType + "> then type: <" + indexSortType + ">";
log.severe(message);
throw new Exception(message);
}
}
}
}
}
}
return sortTypeMap;
}
use of io.zulia.message.ZuliaQuery.FieldSort in project zuliasearch by zuliaio.
the class ZuliaPostSortingComparator method compare.
@Override
public int compare(ScoredResult o1, ScoredResult o2) {
if (fieldSortList == null || fieldSortList.isEmpty()) {
return scoreCompare.compare(o1, o2);
}
int compare = 0;
int sortValueIndex = 0;
SortValues sortValues1 = o1.getSortValues();
SortValues sortValues2 = o2.getSortValues();
for (FieldSort fs : fieldSortList) {
String sortField = fs.getSortField();
FieldConfig.FieldType sortType = sortTypeMap.get(sortField);
if (!ZuliaParser.rewriteLengthFields(sortField).equals(sortField)) {
sortType = FieldConfig.FieldType.NUMERIC_LONG;
}
if (ZuliaConstants.SCORE_FIELD.equals(sortField)) {
if (FieldSort.Direction.DESCENDING.equals(fs.getDirection())) {
compare = scoreCompare.compare(o1, o2);
} else {
compare = reverseScoreCompare.compare(o1, o2);
}
} else {
ZuliaQuery.SortValue sortValue1 = sortValues1.getSortValue(sortValueIndex);
ZuliaQuery.SortValue sortValue2 = sortValues2.getSortValue(sortValueIndex);
if (FieldTypeUtil.isNumericIntFieldType(sortType)) {
Integer a = sortValue1.getExists() ? sortValue1.getIntegerValue() : null;
Integer b = sortValue2.getExists() ? sortValue2.getIntegerValue() : null;
if (!fs.getMissingLast()) {
compare = Comparator.nullsFirst(Integer::compareTo).compare(a, b);
} else {
compare = Comparator.nullsLast(Integer::compareTo).compare(a, b);
}
} else if (FieldTypeUtil.isNumericLongFieldType(sortType)) {
Long a = sortValue1.getExists() ? sortValue1.getLongValue() : null;
Long b = sortValue2.getExists() ? sortValue2.getLongValue() : null;
if (!fs.getMissingLast()) {
compare = Comparator.nullsFirst(Long::compareTo).compare(a, b);
} else {
compare = Comparator.nullsLast(Long::compareTo).compare(a, b);
}
} else if (FieldTypeUtil.isDateFieldType(sortType)) {
Long a = sortValue1.getExists() ? sortValue1.getDateValue() : null;
Long b = sortValue2.getExists() ? sortValue2.getDateValue() : null;
if (!fs.getMissingLast()) {
compare = Comparator.nullsFirst(Long::compareTo).compare(a, b);
} else {
compare = Comparator.nullsLast(Long::compareTo).compare(a, b);
}
} else if (FieldTypeUtil.isNumericFloatFieldType(sortType)) {
Float a = sortValue1.getExists() ? sortValue1.getFloatValue() : null;
Float b = sortValue2.getExists() ? sortValue2.getFloatValue() : null;
if (!fs.getMissingLast()) {
compare = Comparator.nullsFirst(Float::compareTo).compare(a, b);
} else {
compare = Comparator.nullsLast(Float::compareTo).compare(a, b);
}
} else if (FieldTypeUtil.isNumericDoubleFieldType(sortType)) {
Double a = sortValue1.getExists() ? sortValue1.getDoubleValue() : null;
Double b = sortValue2.getExists() ? sortValue2.getDoubleValue() : null;
if (!fs.getMissingLast()) {
compare = Comparator.nullsFirst(Double::compareTo).compare(a, b);
} else {
compare = Comparator.nullsLast(Double::compareTo).compare(a, b);
}
} else {
String a = sortValue1.getExists() ? sortValue1.getStringValue() : null;
String b = sortValue2.getExists() ? sortValue2.getStringValue() : null;
if (!fs.getMissingLast()) {
compare = Comparator.nullsFirst(BytesRef::compareTo).compare(a != null ? new BytesRef(a) : null, b != null ? new BytesRef(b) : null);
} else {
compare = Comparator.nullsLast(BytesRef::compareTo).compare(a != null ? new BytesRef(a) : null, b != null ? new BytesRef(b) : null);
}
}
if (FieldSort.Direction.DESCENDING.equals(fs.getDirection())) {
compare *= -1;
}
}
if (compare != 0) {
return compare;
}
sortValueIndex++;
}
return compare;
}
use of io.zulia.message.ZuliaQuery.FieldSort in project zuliasearch by zuliaio.
the class QueryCombiner method mergeResults.
private List<ScoredResult> mergeResults(int returnedHits, int resultsSize, Map<String, ScoredResult[]> lastIndexResultMap) throws Exception {
List<ScoredResult> results = Collections.emptyList();
boolean sorting = (sortRequest != null && !sortRequest.getFieldSortList().isEmpty());
List<ScoredResult> mergedResults = new ArrayList<>(returnedHits);
for (ShardQueryResponse sr : shardResponses) {
mergedResults.addAll(sr.getScoredResultList());
}
if (!mergedResults.isEmpty()) {
List<FieldSort> fieldSortList = sortRequest != null ? sortRequest.getFieldSortList() : Collections.emptyList();
HashMap<String, FieldConfig.FieldType> sortTypeMap = createSortTypeMap(fieldSortList);
Comparator<ScoredResult> comparator = new ZuliaPostSortingComparator(fieldSortList, sortTypeMap);
mergedResults.sort(comparator);
results = mergedResults.subList(0, resultsSize);
for (ScoredResult sr : results) {
ScoredResult[] lastForShardArr = lastIndexResultMap.get(sr.getIndexName());
lastForShardArr[sr.getShard()] = sr;
}
outside: for (ZuliaIndex index : indexes) {
String indexName = index.getIndexName();
ScoredResult[] lastForShardArr = lastIndexResultMap.get(indexName);
ScoredResult lastForIndex = null;
for (ScoredResult sr : lastForShardArr) {
if (sr != null) {
if (lastForIndex == null) {
lastForIndex = sr;
} else {
if (comparator.compare(sr, lastForIndex) > 0) {
lastForIndex = sr;
}
}
}
}
if (lastForIndex == null) {
// this happen when amount from index is zero
continue;
}
double shardTolerance = index.getShardTolerance();
int numberOfShards = index.getNumberOfShards();
Map<Integer, ShardQueryResponse> shardResponseMap = indexToShardQueryResponseMap.get(indexName);
for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) {
ShardQueryResponse sr = shardResponseMap.get(shardNumber);
if (sr.hasNext()) {
ScoredResult next = sr.getNext();
int compare = comparator.compare(lastForIndex, next);
if (compare > 0) {
if (sorting) {
String msg = "Result set did not return the most relevant sorted documents for index <" + indexName + ">\n";
msg += " Last for index from shard <" + lastForIndex.getShard() + "> has sort values <" + lastForIndex.getSortValues() + ">\n";
msg += " Next for shard <" + next.getShard() + "> has sort values <" + next.getSortValues() + ">\n";
msg += " Last for shards: \n";
msg += " " + Arrays.toString(lastForShardArr) + "\n";
msg += " Results: \n";
msg += " " + results + "\n";
msg += " If this happens frequently increase requestFactor or minShardRequest\n";
msg += " Retrying with full request.\n";
log.severe(msg);
isShort = true;
break outside;
}
double diff = (Math.abs(lastForIndex.getScore() - next.getScore()));
if (diff > shardTolerance) {
String msg = "Result set did not return the most relevant documents for index <" + indexName + "> with shard tolerance <" + shardTolerance + ">\n";
msg += " Last for index from shard <" + lastForIndex.getShard() + "> has score <" + lastForIndex.getScore() + ">\n";
msg += " Next for shard <" + next.getShard() + "> has score <" + next.getScore() + ">\n";
msg += " Last for shards: \n";
msg += " " + Arrays.toString(lastForShardArr) + "\n";
msg += " Results: \n";
msg += " " + results + "\n";
msg += " If this happens frequently increase requestFactor, minShardRequest, or shardTolerance\n";
msg += " Retrying with full request.\n";
log.severe(msg);
isShort = true;
break outside;
}
}
}
}
}
}
return results;
}
Aggregations