Examples with ZuliaIndex io.zulia.server.index.ZuliaIndex used on opensource projects

Example 1 with ZuliaIndex

use of io.zulia.server.index.ZuliaIndex in project zuliasearch by zuliaio.

the class QueryCombiner method createSortTypeMap.

private HashMap<String, FieldConfig.FieldType> createSortTypeMap(List<FieldSort> fieldSortList) throws Exception {
    HashMap<String, FieldConfig.FieldType> sortTypeMap = new HashMap<>();
    if (!fieldSortList.isEmpty()) {
        for (FieldSort fieldSort : fieldSortList) {
            String sortField = fieldSort.getSortField();
            if (ZuliaParser.rewriteLengthFields(sortField).equals(sortField)) {
                for (ZuliaIndex index : indexes) {
                    FieldConfig.FieldType currentSortType = sortTypeMap.get(sortField);
                    FieldConfig.FieldType indexSortType = index.getSortFieldType(sortField);
                    if (currentSortType == null) {
                        sortTypeMap.put(sortField, indexSortType);
                    } else {
                        if (!currentSortType.equals(indexSortType)) {
                            log.severe("Sort fields must be defined the same in all indexes searched in a single query");
                            String message = "Cannot sort on field <" + sortField + ">: found type: <" + currentSortType + "> then type: <" + indexSortType + ">";
                            log.severe(message);
                            throw new Exception(message);
                        }
                    }
                }
            }
        }
    }
    return sortTypeMap;
}

Example 2 with ZuliaIndex

use of io.zulia.server.index.ZuliaIndex in project zuliasearch by zuliaio.

the class QueryRequestFederator method internalQuery.

public static InternalQueryResponse internalQuery(Collection<ZuliaIndex> indexes, InternalQueryRequest request, Map<String, Query> queryMap) throws Exception {
    InternalQueryResponse.Builder internalQueryResponseBuilder = InternalQueryResponse.newBuilder();
    for (ZuliaIndex index : indexes) {
        IndexShardResponse isr = index.internalQuery(queryMap.get(index.getIndexName()), request);
        internalQueryResponseBuilder.addIndexShardResponse(isr);
    }
    return internalQueryResponseBuilder.build();
}

Example 3 with ZuliaIndex

use of io.zulia.server.index.ZuliaIndex in project zuliasearch by zuliaio.

the class QueryCombiner method mergeResults.

private List<ScoredResult> mergeResults(int returnedHits, int resultsSize, Map<String, ScoredResult[]> lastIndexResultMap) throws Exception {
    List<ScoredResult> results = Collections.emptyList();
    boolean sorting = (sortRequest != null && !sortRequest.getFieldSortList().isEmpty());
    List<ScoredResult> mergedResults = new ArrayList<>(returnedHits);
    for (ShardQueryResponse sr : shardResponses) {
        mergedResults.addAll(sr.getScoredResultList());
    }
    if (!mergedResults.isEmpty()) {
        List<FieldSort> fieldSortList = sortRequest != null ? sortRequest.getFieldSortList() : Collections.emptyList();
        HashMap<String, FieldConfig.FieldType> sortTypeMap = createSortTypeMap(fieldSortList);
        Comparator<ScoredResult> comparator = new ZuliaPostSortingComparator(fieldSortList, sortTypeMap);
        mergedResults.sort(comparator);
        results = mergedResults.subList(0, resultsSize);
        for (ScoredResult sr : results) {
            ScoredResult[] lastForShardArr = lastIndexResultMap.get(sr.getIndexName());
            lastForShardArr[sr.getShard()] = sr;
        }
        outside: for (ZuliaIndex index : indexes) {
            String indexName = index.getIndexName();
            ScoredResult[] lastForShardArr = lastIndexResultMap.get(indexName);
            ScoredResult lastForIndex = null;
            for (ScoredResult sr : lastForShardArr) {
                if (sr != null) {
                    if (lastForIndex == null) {
                        lastForIndex = sr;
                    } else {
                        if (comparator.compare(sr, lastForIndex) > 0) {
                            lastForIndex = sr;
                        }
                    }
                }
            }
            if (lastForIndex == null) {
                // this happen when amount from index is zero
                continue;
            }
            double shardTolerance = index.getShardTolerance();
            int numberOfShards = index.getNumberOfShards();
            Map<Integer, ShardQueryResponse> shardResponseMap = indexToShardQueryResponseMap.get(indexName);
            for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) {
                ShardQueryResponse sr = shardResponseMap.get(shardNumber);
                if (sr.hasNext()) {
                    ScoredResult next = sr.getNext();
                    int compare = comparator.compare(lastForIndex, next);
                    if (compare > 0) {
                        if (sorting) {
                            String msg = "Result set did not return the most relevant sorted documents for index <" + indexName + ">\n";
                            msg += "    Last for index from shard <" + lastForIndex.getShard() + "> has sort values <" + lastForIndex.getSortValues() + ">\n";
                            msg += "    Next for shard <" + next.getShard() + ">  has sort values <" + next.getSortValues() + ">\n";
                            msg += "    Last for shards: \n";
                            msg += "      " + Arrays.toString(lastForShardArr) + "\n";
                            msg += "    Results: \n";
                            msg += "      " + results + "\n";
                            msg += "    If this happens frequently increase requestFactor or minShardRequest\n";
                            msg += "    Retrying with full request.\n";
                            log.severe(msg);
                            isShort = true;
                            break outside;
                        }
                        double diff = (Math.abs(lastForIndex.getScore() - next.getScore()));
                        if (diff > shardTolerance) {
                            String msg = "Result set did not return the most relevant documents for index <" + indexName + "> with shard tolerance <" + shardTolerance + ">\n";
                            msg += "    Last for index from shard <" + lastForIndex.getShard() + "> has score <" + lastForIndex.getScore() + ">\n";
                            msg += "    Next for shard <" + next.getShard() + "> has score <" + next.getScore() + ">\n";
                            msg += "    Last for shards: \n";
                            msg += "      " + Arrays.toString(lastForShardArr) + "\n";
                            msg += "    Results: \n";
                            msg += "      " + results + "\n";
                            msg += "    If this happens frequently increase requestFactor, minShardRequest, or shardTolerance\n";
                            msg += "    Retrying with full request.\n";
                            log.severe(msg);
                            isShort = true;
                            break outside;
                        }
                    }
                }
            }
        }
    }
    return results;
}

Example 4 with ZuliaIndex

use of io.zulia.server.index.ZuliaIndex in project zuliasearch by zuliaio.

the class QueryCombiner method validate.

private void validate() throws Exception {
    for (InternalQueryResponse iqr : responses) {
        for (IndexShardResponse isr : iqr.getIndexShardResponseList()) {
            String indexName = isr.getIndexName();
            if (!indexToShardQueryResponseMap.containsKey(indexName)) {
                indexToShardQueryResponseMap.put(indexName, new HashMap<>());
            }
            for (ShardQueryResponse sr : isr.getShardQueryResponseList()) {
                int shardNumber = sr.getShardNumber();
                Map<Integer, ShardQueryResponse> shardResponseMap = indexToShardQueryResponseMap.get(indexName);
                if (shardResponseMap.containsKey(shardNumber)) {
                    throw new Exception("Shard <" + shardNumber + "> is repeated for <" + indexName + ">");
                } else {
                    shardResponseMap.put(shardNumber, sr);
                    shardResponses.add(sr);
                }
            }
        }
    }
    for (ZuliaIndex index : indexes) {
        int numberOfShards = index.getNumberOfShards();
        Map<Integer, ShardQueryResponse> shardResponseMap = indexToShardQueryResponseMap.get(index.getIndexName());
        if (shardResponseMap == null) {
            throw new Exception("Missing index <" + index.getIndexName() + "> in response");
        }
        if (shardResponseMap.size() != numberOfShards) {
            throw new Exception("Found <" + shardResponseMap.size() + "> expected <" + numberOfShards + ">");
        }
        for (int shardNumber = 0; shardNumber < numberOfShards; shardNumber++) {
            if (!shardResponseMap.containsKey(shardNumber)) {
                throw new Exception("Missing shard <" + shardNumber + ">");
            }
        }
    }
}