Search in sources :

Example 1 with PluginDriverDTOList

use of io.openk9.plugin.driver.manager.model.PluginDriverDTOList in project openk9 by smclab.

the class SuggestionsHTTPHandler method _getSuggestionsResponse.

private SuggestionsResponse _getSuggestionsResponse(List<Datasource> datasourceList, PluginDriverDTOList pluginDriverDTOList, SearchRequest searchRequest, SearchResponse searchResponse, List<SuggestionCategoryField> fields, Map<String, String[]> entityMap) {
    Aggregations aggregations = searchResponse.getAggregations();
    if (aggregations == null) {
        return SuggestionsResponse.of(List.of(), null);
    }
    Map<String, Aggregation> aggregationMap = aggregations.asMap();
    if (!aggregationMap.containsKey("composite")) {
        return SuggestionsResponse.of(List.of(), null);
    }
    CompositeAggregation compositeAggregation = (CompositeAggregation) aggregationMap.get("composite");
    Map<String, Long> fieldNameCategoryIdMap = fields.stream().collect(Collectors.toMap(SuggestionCategoryField::getFieldName, SuggestionCategoryField::getCategoryId, (a1, a2) -> a2));
    Long datasourceIdCategoryId = fieldNameCategoryIdMap.getOrDefault("datasourceId", 1L);
    Long entityIdCategoryId = fieldNameCategoryIdMap.getOrDefault("entities.id", 2L);
    Long entitiesContextCategoryId = fieldNameCategoryIdMap.getOrDefault("entities.context", entityIdCategoryId);
    Long documentTypesCategoryId = fieldNameCategoryIdMap.getOrDefault("documentTypes", 4L);
    List<? extends CompositeAggregation.Bucket> buckets = compositeAggregation.getBuckets();
    LinkedList<Suggestions> suggestions = new LinkedList<>();
    String suggestKeyword = searchRequest.getSuggestKeyword();
    BiConsumer<String, Suggestions> addSuggestions;
    if (suggestKeyword != null) {
        addSuggestions = (key, sugg) -> {
            if (!suggestions.contains(sugg)) {
                if (key.contains(suggestKeyword)) {
                    suggestions.addFirst(sugg);
                }
            }
        };
    } else {
        addSuggestions = (key, sugg) -> {
            if (!suggestions.contains(sugg)) {
                suggestions.add(sugg);
            }
        };
    }
    for (CompositeAggregation.Bucket bucket : buckets) {
        Map<String, Object> keys = new HashMap<>(bucket.getKey());
        for (Map.Entry<String, Object> entry : keys.entrySet()) {
            String key = entry.getKey();
            String value = (String) entry.getValue();
            if (value == null) {
                continue;
            }
            switch(key) {
                case "datasourceId":
                    long datasourceIdL = Long.parseLong(value);
                    _datasource(datasourceList, pluginDriverDTOList, datasourceIdCategoryId, addSuggestions, datasourceIdL);
                    break;
                case "entities.context":
                    break;
                case "entities.id":
                    String[] typeName = entityMap.get(value);
                    if (typeName != null) {
                        String type = typeName[0];
                        String name = typeName[1];
                        String entitiesContext = (String) keys.get("entities.context");
                        if (entitiesContext != null) {
                            addSuggestions.accept(name, Suggestions.entity(value, entitiesContextCategoryId, type, name, entitiesContext));
                        } else {
                            addSuggestions.accept(name, Suggestions.entity(value, entityIdCategoryId, type, name));
                        }
                    }
                    break;
                case "documentTypes":
                    addSuggestions.accept(value, Suggestions.docType(value, documentTypesCategoryId));
                    break;
                default:
                    Long textCategoryId = fieldNameCategoryIdMap.getOrDefault(key, 5L);
                    addSuggestions.accept(value, Suggestions.text(value, textCategoryId, key));
            }
        }
    }
    Map<String, Object> map = compositeAggregation.afterKey();
    String afterKey = null;
    int[] range = searchRequest.getRange();
    if (map != null) {
        afterKey = _jsonFactory.toJson(map);
        afterKey = Base64.getEncoder().encodeToString(afterKey.getBytes(StandardCharsets.UTF_8));
    }
    return SuggestionsResponse.of(suggestions, afterKey);
}
Also used : PluginDriverDTO(io.openk9.plugin.driver.manager.model.PluginDriverDTO) HttpUtil(io.openk9.http.util.HttpUtil) PluginDriverDTOList(io.openk9.plugin.driver.manager.model.PluginDriverDTOList) LoggerFactory(org.slf4j.LoggerFactory) SuggestionsResponse(io.openk9.search.query.internal.response.SuggestionsResponse) QueryParser(io.openk9.search.api.query.QueryParser) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) SearchToken(io.openk9.search.api.query.SearchToken) JsonFactory(io.openk9.json.api.JsonFactory) Map(java.util.Map) SearchResponse(org.elasticsearch.action.search.SearchResponse) PluginDriverManagerClient(io.openk9.plugin.driver.manager.client.api.PluginDriverManagerClient) SearchHit(org.elasticsearch.search.SearchHit) Tenant(io.openk9.model.Tenant) Aggregations(org.elasticsearch.search.aggregations.Aggregations) DatasourceClient(io.openk9.datasource.client.api.DatasourceClient) ReferencePolicy(org.osgi.service.component.annotations.ReferencePolicy) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) Objects(java.util.Objects) HttpServerRequest(reactor.netty.http.server.HttpServerRequest) Base64(java.util.Base64) List(java.util.List) ReferencePolicyOption(org.osgi.service.component.annotations.ReferencePolicyOption) SuggestionCategoryField(io.openk9.model.SuggestionCategoryField) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) HttpServerRoutes(reactor.netty.http.server.HttpServerRoutes) HttpServerResponse(reactor.netty.http.server.HttpServerResponse) Search(io.openk9.search.client.api.Search) Tuple2(reactor.util.function.Tuple2) HashMap(java.util.HashMap) HttpResponseWriter(io.openk9.http.util.HttpResponseWriter) Function(java.util.function.Function) Component(org.osgi.service.component.annotations.Component) CompositeAggregation(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation) CompositeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) BiConsumer(java.util.function.BiConsumer) Aggregation(org.elasticsearch.search.aggregations.Aggregation) Activate(org.osgi.service.component.annotations.Activate) LinkedList(java.util.LinkedList) RouterHandler(io.openk9.http.web.RouterHandler) CompositeValuesSourceBuilder(org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder) Logger(org.slf4j.Logger) Datasource(io.openk9.model.Datasource) Suggestions(io.openk9.search.query.internal.response.suggestions.Suggestions) AggregationBuilders(org.elasticsearch.search.aggregations.AggregationBuilders) Mono(reactor.core.publisher.Mono) TermsValuesSourceBuilder(org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder) ReferenceCardinality(org.osgi.service.component.annotations.ReferenceCardinality) SearchRequest(io.openk9.search.api.query.SearchRequest) SearchTokenizer(io.openk9.search.api.query.SearchTokenizer) Modified(org.osgi.service.component.annotations.Modified) Reference(org.osgi.service.component.annotations.Reference) CompositeAggregation(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation) HashMap(java.util.HashMap) Aggregations(org.elasticsearch.search.aggregations.Aggregations) LinkedList(java.util.LinkedList) CompositeAggregation(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation) Aggregation(org.elasticsearch.search.aggregations.Aggregation) Suggestions(io.openk9.search.query.internal.response.suggestions.Suggestions) Map(java.util.Map) HashMap(java.util.HashMap)

Example 2 with PluginDriverDTOList

use of io.openk9.plugin.driver.manager.model.PluginDriverDTOList in project openk9 by smclab.

the class SuggestionsHTTPHandler method searchHitToResponseMono.

@Override
protected Mono<Object> searchHitToResponseMono(Tenant tenant, List<Datasource> datasourceList, PluginDriverDTOList pluginDriverDTOList, HttpServerRequest httpServerRequest, SearchRequest searchRequest, SearchResponse searchResponse) {
    Long suggestionCategoryId = searchRequest.getSuggestionCategoryId();
    Mono<List<SuggestionCategoryField>> suggestionCategoryFields;
    if (suggestionCategoryId == null) {
        suggestionCategoryFields = _datasourceClient.findSuggestionCategoryFieldsByTenantId(tenant.getTenantId());
    } else {
        suggestionCategoryFields = _datasourceClient.findSuggestionCategoryFieldsByTenantIdAndCategoryId(tenant.getTenantId(), suggestionCategoryId);
    }
    if (_enableEntityAggregation) {
        return _search.search(factory -> {
            org.elasticsearch.action.search.SearchRequest searchRequestEntity = factory.createSearchRequestEntity(tenant.getTenantId());
            Aggregations aggregations = searchResponse.getAggregations();
            CompositeAggregation compositeAggregation = aggregations.get("composite");
            List<? extends CompositeAggregation.Bucket> buckets = compositeAggregation.getBuckets();
            BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
            buckets.stream().map(bucket -> (String) bucket.getKey().get("entities.id")).filter(Objects::nonNull).distinct().forEach(entityId -> boolQueryBuilder.should(QueryBuilders.matchQuery("id", entityId)));
            if (_log.isDebugEnabled()) {
                _log.debug("entities query: " + boolQueryBuilder);
            }
            SearchSourceBuilder ssb = new SearchSourceBuilder();
            ssb.query(boolQueryBuilder);
            ssb.size(1000);
            ssb.fetchSource(new String[] { "name", "id", "type" }, null);
            return searchRequestEntity.source(ssb);
        }).flatMap(entityResponse -> suggestionCategoryFields.map(fields -> {
            Map<String, String[]> entityMap = new HashMap<>();
            for (SearchHit hit : entityResponse.getHits()) {
                Map<String, Object> sourceAsMap = hit.getSourceAsMap();
                String name = (String) sourceAsMap.get("name");
                String type = (String) sourceAsMap.get("type");
                String entityId = (String) sourceAsMap.get("id");
                entityMap.put(entityId, new String[] { type, name });
            }
            return _getSuggestionsResponse(datasourceList, pluginDriverDTOList, searchRequest, searchResponse, fields, entityMap);
        }));
    } else {
        return suggestionCategoryFields.map(fields -> _getSuggestionsResponse(datasourceList, pluginDriverDTOList, searchRequest, searchResponse, fields, Map.of()));
    }
}
Also used : PluginDriverDTO(io.openk9.plugin.driver.manager.model.PluginDriverDTO) HttpUtil(io.openk9.http.util.HttpUtil) PluginDriverDTOList(io.openk9.plugin.driver.manager.model.PluginDriverDTOList) LoggerFactory(org.slf4j.LoggerFactory) SuggestionsResponse(io.openk9.search.query.internal.response.SuggestionsResponse) QueryParser(io.openk9.search.api.query.QueryParser) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) SearchToken(io.openk9.search.api.query.SearchToken) JsonFactory(io.openk9.json.api.JsonFactory) Map(java.util.Map) SearchResponse(org.elasticsearch.action.search.SearchResponse) PluginDriverManagerClient(io.openk9.plugin.driver.manager.client.api.PluginDriverManagerClient) SearchHit(org.elasticsearch.search.SearchHit) Tenant(io.openk9.model.Tenant) Aggregations(org.elasticsearch.search.aggregations.Aggregations) DatasourceClient(io.openk9.datasource.client.api.DatasourceClient) ReferencePolicy(org.osgi.service.component.annotations.ReferencePolicy) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) Objects(java.util.Objects) HttpServerRequest(reactor.netty.http.server.HttpServerRequest) Base64(java.util.Base64) List(java.util.List) ReferencePolicyOption(org.osgi.service.component.annotations.ReferencePolicyOption) SuggestionCategoryField(io.openk9.model.SuggestionCategoryField) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) HttpServerRoutes(reactor.netty.http.server.HttpServerRoutes) HttpServerResponse(reactor.netty.http.server.HttpServerResponse) Search(io.openk9.search.client.api.Search) Tuple2(reactor.util.function.Tuple2) HashMap(java.util.HashMap) HttpResponseWriter(io.openk9.http.util.HttpResponseWriter) Function(java.util.function.Function) Component(org.osgi.service.component.annotations.Component) CompositeAggregation(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation) CompositeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) BiConsumer(java.util.function.BiConsumer) Aggregation(org.elasticsearch.search.aggregations.Aggregation) Activate(org.osgi.service.component.annotations.Activate) LinkedList(java.util.LinkedList) RouterHandler(io.openk9.http.web.RouterHandler) CompositeValuesSourceBuilder(org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder) Logger(org.slf4j.Logger) Datasource(io.openk9.model.Datasource) Suggestions(io.openk9.search.query.internal.response.suggestions.Suggestions) AggregationBuilders(org.elasticsearch.search.aggregations.AggregationBuilders) Mono(reactor.core.publisher.Mono) TermsValuesSourceBuilder(org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder) ReferenceCardinality(org.osgi.service.component.annotations.ReferenceCardinality) SearchRequest(io.openk9.search.api.query.SearchRequest) SearchTokenizer(io.openk9.search.api.query.SearchTokenizer) Modified(org.osgi.service.component.annotations.Modified) Reference(org.osgi.service.component.annotations.Reference) SearchRequest(io.openk9.search.api.query.SearchRequest) CompositeAggregation(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation) SearchHit(org.elasticsearch.search.SearchHit) Aggregations(org.elasticsearch.search.aggregations.Aggregations) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) PluginDriverDTOList(io.openk9.plugin.driver.manager.model.PluginDriverDTOList) List(java.util.List) LinkedList(java.util.LinkedList) Map(java.util.Map) HashMap(java.util.HashMap)

Example 3 with PluginDriverDTOList

use of io.openk9.plugin.driver.manager.model.PluginDriverDTOList in project openk9 by smclab.

the class BaseSearchHTTPHandler method _toQuerySearchRequest.

private Mono<SearchResponse> _toQuerySearchRequest(Tenant tenant, List<Datasource> datasources, PluginDriverDTOList pdDTOList, SearchRequest searchRequest, HttpServerRequest httpRequest) {
    return Mono.defer(() -> {
        List<PluginDriverDTO> pluginDriverDTOList = pdDTOList.getPluginDriverDTOList();
        Map<String, List<SearchToken>> tokenTypeGroup = searchRequest.getSearchQuery().stream().collect(Collectors.groupingBy(SearchToken::getTokenType));
        List<SearchToken> datasource = tokenTypeGroup.get("DATASOURCE");
        Stream<PluginDriverDTO> documentTypeStream = pluginDriverDTOList.stream();
        if (datasource != null) {
            List<String> datasourceValues = datasource.stream().map(SearchToken::getValues).flatMap(Arrays::stream).distinct().collect(Collectors.toList());
            documentTypeStream = documentTypeStream.filter(entry -> datasourceValues.contains(entry.getName()));
        }
        List<PluginDriverDTO> documentTypeList = documentTypeStream.collect(Collectors.toList());
        QueryParser queryParser = _queryParsers.stream().reduce(QueryParser.NOTHING, QueryParser::andThen);
        return queryParser.apply(createQueryParserContext(tenant, datasources, httpRequest, tokenTypeGroup, documentTypeList)).flatMap(boolQueryBuilderConsumer -> _search.flatMapSearch(factory -> {
            long tenantId = tenant.getTenantId();
            if (documentTypeList.isEmpty()) {
                return Mono.just(SearchUtil.EMPTY_SEARCH_REQUEST);
            }
            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
            boolQueryBuilderConsumer.accept(boolQuery);
            org.elasticsearch.action.search.SearchRequest elasticSearchQuery;
            if (datasource != null) {
                String[] indexNames = documentTypeList.stream().map(PluginDriverDTO::getName).distinct().toArray(String[]::new);
                elasticSearchQuery = factory.createSearchRequestData(tenantId, indexNames);
            } else {
                elasticSearchQuery = factory.createSearchRequestData(tenantId, "*");
            }
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.fetchSource(includeFields(), excludeFields());
            searchSourceBuilder.query(boolQuery);
            searchSourceBuilder.trackTotalHits(true);
            return customizeSearchSourceBuilderMono(tenant, datasources, searchRequest, documentTypeList, searchSourceBuilder, elasticSearchQuery);
        }));
    });
}
Also used : HttpServerResponse(reactor.netty.http.server.HttpServerResponse) PluginDriverDTO(io.openk9.plugin.driver.manager.model.PluginDriverDTO) Arrays(java.util.Arrays) SearchUtil(io.openk9.search.client.api.util.SearchUtil) SearchHits(org.elasticsearch.search.SearchHits) PluginDriverDTOList(io.openk9.plugin.driver.manager.model.PluginDriverDTOList) LoggerFactory(org.slf4j.LoggerFactory) Tuples(reactor.util.function.Tuples) Search(io.openk9.search.client.api.Search) Tuple2(reactor.util.function.Tuple2) HashMap(java.util.HashMap) QueryParser(io.openk9.search.api.query.QueryParser) HttpResponseWriter(io.openk9.http.util.HttpResponseWriter) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) SearchToken(io.openk9.search.api.query.SearchToken) JsonFactory(io.openk9.json.api.JsonFactory) ArrayList(java.util.ArrayList) HighlightBuilder(org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder) ReactorNettyUtils(io.openk9.reactor.netty.util.ReactorNettyUtils) Text(org.elasticsearch.common.text.Text) Map(java.util.Map) SearchResponse(org.elasticsearch.action.search.SearchResponse) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) HighlightField(org.elasticsearch.search.fetch.subphase.highlight.HighlightField) PluginDriverManagerClient(io.openk9.plugin.driver.manager.client.api.PluginDriverManagerClient) RouterHandler(io.openk9.http.web.RouterHandler) SearchHit(org.elasticsearch.search.SearchHit) Tenant(io.openk9.model.Tenant) Logger(org.slf4j.Logger) Datasource(io.openk9.model.Datasource) Publisher(org.reactivestreams.Publisher) Collection(java.util.Collection) DocumentTypeDTO(io.openk9.plugin.driver.manager.model.DocumentTypeDTO) DatasourceClient(io.openk9.datasource.client.api.DatasourceClient) Mono(reactor.core.publisher.Mono) SearchKeywordDTO(io.openk9.plugin.driver.manager.model.SearchKeywordDTO) Collectors(java.util.stream.Collectors) TotalHits(org.apache.lucene.search.TotalHits) SearchRequest(io.openk9.search.api.query.SearchRequest) HttpServerRequest(reactor.netty.http.server.HttpServerRequest) List(java.util.List) HttpHandler(io.openk9.http.web.HttpHandler) Stream(java.util.stream.Stream) SearchTokenizer(io.openk9.search.api.query.SearchTokenizer) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) Response(io.openk9.search.query.internal.response.Response) SearchRequest(io.openk9.search.api.query.SearchRequest) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) QueryParser(io.openk9.search.api.query.QueryParser) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) SearchToken(io.openk9.search.api.query.SearchToken) PluginDriverDTOList(io.openk9.plugin.driver.manager.model.PluginDriverDTOList) ArrayList(java.util.ArrayList) List(java.util.List) Arrays(java.util.Arrays) PluginDriverDTO(io.openk9.plugin.driver.manager.model.PluginDriverDTO)

Aggregations

DatasourceClient (io.openk9.datasource.client.api.DatasourceClient)3 HttpResponseWriter (io.openk9.http.util.HttpResponseWriter)3 RouterHandler (io.openk9.http.web.RouterHandler)3 JsonFactory (io.openk9.json.api.JsonFactory)3 Datasource (io.openk9.model.Datasource)3 Tenant (io.openk9.model.Tenant)3 PluginDriverManagerClient (io.openk9.plugin.driver.manager.client.api.PluginDriverManagerClient)3 PluginDriverDTO (io.openk9.plugin.driver.manager.model.PluginDriverDTO)3 PluginDriverDTOList (io.openk9.plugin.driver.manager.model.PluginDriverDTOList)3 QueryParser (io.openk9.search.api.query.QueryParser)3 SearchRequest (io.openk9.search.api.query.SearchRequest)3 SearchToken (io.openk9.search.api.query.SearchToken)3 SearchTokenizer (io.openk9.search.api.query.SearchTokenizer)3 Search (io.openk9.search.client.api.Search)3 HashMap (java.util.HashMap)3 List (java.util.List)3 Map (java.util.Map)3 Collectors (java.util.stream.Collectors)3 SearchResponse (org.elasticsearch.action.search.SearchResponse)3 BoolQueryBuilder (org.elasticsearch.index.query.BoolQueryBuilder)3