use of io.zulia.message.ZuliaQuery.IndexShardResponse in project zuliasearch by zuliaio.
the class QueryRequestFederator method internalQuery.
public static InternalQueryResponse internalQuery(Collection<ZuliaIndex> indexes, InternalQueryRequest request, Map<String, Query> queryMap) throws Exception {
InternalQueryResponse.Builder internalQueryResponseBuilder = InternalQueryResponse.newBuilder();
for (ZuliaIndex index : indexes) {
IndexShardResponse isr = index.internalQuery(queryMap.get(index.getIndexName()), request);
internalQueryResponseBuilder.addIndexShardResponse(isr);
}
return internalQueryResponseBuilder.build();
}
use of io.zulia.message.ZuliaQuery.IndexShardResponse in project zuliasearch by zuliaio.
the class QueryCombiner method validate.
private void validate() throws Exception {
for (InternalQueryResponse iqr : responses) {
for (IndexShardResponse isr : iqr.getIndexShardResponseList()) {
String indexName = isr.getIndexName();
if (!indexToShardQueryResponseMap.containsKey(indexName)) {
indexToShardQueryResponseMap.put(indexName, new HashMap<>());
}
for (ShardQueryResponse sr : isr.getShardQueryResponseList()) {
int shardNumber = sr.getShardNumber();
Map<Integer, ShardQueryResponse> shardResponseMap = indexToShardQueryResponseMap.get(indexName);
if (shardResponseMap.containsKey(shardNumber)) {
throw new Exception("Shard <" + shardNumber + "> is repeated for <" + indexName + ">");
} else {
shardResponseMap.put(shardNumber, sr);
shardResponses.add(sr);
}
}
}
}
for (ZuliaIndex index : indexes) {
int numberOfShards = index.getNumberOfShards();
Map<Integer, ShardQueryResponse> shardResponseMap = indexToShardQueryResponseMap.get(index.getIndexName());
if (shardResponseMap == null) {
throw new Exception("Missing index <" + index.getIndexName() + "> in response");
}
if (shardResponseMap.size() != numberOfShards) {
throw new Exception("Found <" + shardResponseMap.size() + "> expected <" + numberOfShards + ">");
}
for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) {
if (!shardResponseMap.containsKey(shardNumber)) {
throw new Exception("Missing shard <" + shardNumber + ">");
}
}
}
}
use of io.zulia.message.ZuliaQuery.IndexShardResponse in project zuliasearch by zuliaio.
the class ZuliaIndex method internalQuery.
public IndexShardResponse internalQuery(Query query, final InternalQueryRequest internalQueryRequest) throws Exception {
QueryRequest queryRequest = internalQueryRequest.getQueryRequest();
Set<ZuliaShard> shardsForQuery = new HashSet<>();
for (IndexRouting indexRouting : internalQueryRequest.getIndexRoutingList()) {
if (indexRouting.getIndex().equals(indexName)) {
List<ZuliaShard> shardsFromRouting = getShardsFromRouting(indexRouting, queryRequest.getMasterSlaveSettings());
shardsForQuery.addAll(shardsFromRouting);
}
}
int amount = queryRequest.getAmount() + queryRequest.getStart();
if (indexConfig.getNumberOfShards() != 1) {
if (!queryRequest.getFetchFull() && (amount > 0)) {
amount = (int) (((amount / numberOfShards) + indexConfig.getIndexSettings().getMinShardRequest()) * indexConfig.getIndexSettings().getRequestFactor());
}
}
final int requestedAmount = amount;
final HashMap<Integer, FieldDoc> lastScoreDocMap = new HashMap<>();
FieldDoc after;
ZuliaQuery.LastResult lr = queryRequest.getLastResult();
for (ZuliaQuery.LastIndexResult lir : lr.getLastIndexResultList()) {
if (indexName.equals(lir.getIndexName())) {
for (ZuliaQuery.ScoredResult sr : lir.getLastForShardList()) {
int luceneShardId = sr.getLuceneShardId();
float score = sr.getScore();
SortRequest sortRequest = queryRequest.getSortRequest();
Object[] sortTerms = new Object[sortRequest.getFieldSortCount()];
int sortTermsIndex = 0;
ZuliaQuery.SortValues sortValues = sr.getSortValues();
for (ZuliaQuery.FieldSort fs : sortRequest.getFieldSortList()) {
String sortField = fs.getSortField();
FieldConfig.FieldType sortType = indexConfig.getFieldTypeForSortField(sortField);
if (!ZuliaParser.rewriteLengthFields(sortField).equals(sortField)) {
sortType = FieldConfig.FieldType.NUMERIC_LONG;
}
if (sortType == null) {
throw new Exception(sortField + " is not defined as a sortable field");
}
ZuliaQuery.SortValue sortValue = sortValues.getSortValue(sortTermsIndex);
if (ZuliaConstants.SCORE_FIELD.equals(sortField)) {
sortTerms[sortTermsIndex] = sortValue.getFloatValue();
} else if (sortValue.getExists()) {
if (FieldTypeUtil.isNumericOrDateFieldType(sortType)) {
if (FieldTypeUtil.isNumericIntFieldType(sortType)) {
sortTerms[sortTermsIndex] = sortValue.getIntegerValue();
} else if (FieldTypeUtil.isNumericLongFieldType(sortType)) {
sortTerms[sortTermsIndex] = sortValue.getLongValue();
} else if (FieldTypeUtil.isNumericFloatFieldType(sortType)) {
sortTerms[sortTermsIndex] = sortValue.getFloatValue();
} else if (FieldTypeUtil.isNumericDoubleFieldType(sortType)) {
sortTerms[sortTermsIndex] = sortValue.getDoubleValue();
} else if (FieldTypeUtil.isDateFieldType(sortType)) {
sortTerms[sortTermsIndex] = sortValue.getDateValue();
} else {
throw new Exception("Invalid numeric sort type <" + sortType + "> for sort field <" + sortField + ">");
}
} else {
// string
sortTerms[sortTermsIndex] = new BytesRef(sortValue.getStringValue());
}
} else {
sortTerms[sortTermsIndex] = null;
}
sortTermsIndex++;
}
after = new FieldDoc(luceneShardId, score, sortTerms, sr.getShard());
lastScoreDocMap.put(sr.getShard(), after);
}
}
}
Map<String, Similarity> fieldSimilarityMap = new HashMap<>();
for (FieldSimilarity fieldSimilarity : queryRequest.getFieldSimilarityList()) {
fieldSimilarityMap.put(fieldSimilarity.getField(), fieldSimilarity.getSimilarity());
}
for (ZuliaQuery.Query cosineSimQuery : queryRequest.getQueryList()) {
if (cosineSimQuery.getQueryType() == ZuliaQuery.Query.QueryType.VECTOR) {
for (String field : cosineSimQuery.getQfList()) {
io.zulia.message.ZuliaIndex.Superbit superbitConfig = indexConfig.getSuperBitConfigForField(field);
int sigLength = superbitConfig.getInputDim() * superbitConfig.getBatches();
for (int i = 0; i < sigLength; i++) {
String fieldName = ZuliaConstants.SUPERBIT_PREFIX + "." + field + "." + i;
fieldSimilarityMap.put(fieldName, Similarity.CONSTANT);
}
}
}
}
IndexShardResponse.Builder builder = IndexShardResponse.newBuilder();
List<Future<ShardQueryResponse>> responses = new ArrayList<>();
for (final ZuliaShard shard : shardsForQuery) {
Future<ShardQueryResponse> response = shardPool.submit(() -> {
QueryCacheKey queryCacheKey = null;
if (!queryRequest.getDontCache()) {
queryCacheKey = new QueryCacheKey(queryRequest);
}
return shard.queryShard(query, fieldSimilarityMap, requestedAmount, lastScoreDocMap.get(shard.getShardNumber()), queryRequest.getFacetRequest(), queryRequest.getSortRequest(), queryCacheKey, queryRequest.getResultFetchType(), queryRequest.getDocumentFieldsList(), queryRequest.getDocumentMaskedFieldsList(), queryRequest.getHighlightRequestList(), queryRequest.getAnalysisRequestList(), queryRequest.getDebug());
});
responses.add(response);
}
for (Future<ShardQueryResponse> response : responses) {
try {
ShardQueryResponse rs = response.get();
builder.addShardQueryResponse(rs);
} catch (ExecutionException e) {
Throwable t = e.getCause();
if (t instanceof OutOfMemoryError) {
throw (OutOfMemoryError) t;
}
throw ((Exception) e.getCause());
}
}
builder.setIndexName(indexName);
return builder.build();
}
Aggregations