Search in sources :

Example 1 with Datasource

use of io.openk9.datasource.model.Datasource in project openk9 by smclab.

the class GetOrAddEntitiesConsumer method apply.

public Mono<ObjectNode> apply(ObjectNode objectNode) {
    return Mono.defer(() -> {
        ObjectNode datasourceContextJson = objectNode.get("datasourceContext").toObjectNode();
        long datasourceId = datasourceContextJson.get("datasource").get("datasourceId").asLong();
        long tenantId = datasourceContextJson.get("tenant").get("tenantId").asLong();
        JsonNode entities = objectNode.remove("entities");
        Mono<ArrayNode> entitiesField;
        if (entities.size() == 0) {
            entitiesField = Mono.just(_jsonFactory.createArrayNode());
        } else {
            ObjectNode responseJson = _jsonFactory.createObjectNode();
            responseJson.put("entities", entities);
            responseJson.put("tenantId", tenantId);
            responseJson.put("datasourceId", datasourceId);
            Request request = _jsonFactory.fromJson(responseJson.toString(), Request.class);
            List<RequestContext> requestContextList = request.getEntities().stream().map(entityRequest -> RequestContext.builder().current(entityRequest).tenantId(request.getTenantId()).datasourceId(request.getDatasourceId()).rest(request.getEntities().stream().filter(er -> er != entityRequest).collect(Collectors.toList())).build()).collect(Collectors.toList());
            Mono<List<EntityContext>> disambiguateListMono = GetOrAddEntities.stopWatch("disambiguate-all-entities", Flux.fromIterable(requestContextList).flatMap(requestContext -> GetOrAddEntities.stopWatch("disambiguate-" + requestContext.getCurrent().getName(), Mono.<EntityContext>create(fluxSink -> _startDisambiguation.disambiguate(requestContext, fluxSink)))).collectList());
            Mono<ResponseList> writeRelations = disambiguateListMono.flatMap(entityContexts -> GetOrAddEntities.stopWatch("write-relations", writeRelations(entityContexts)));
            Mono<ResponseList> responseListWrapper = _transactional ? _graphClient.makeTransactional(writeRelations) : writeRelations;
            entitiesField = responseListWrapper.map(responseListDTO -> {
                List<Response> responseList = responseListDTO.getResponse();
                ArrayNode entitiesArrayNode = entities.toArrayNode();
                ArrayNode arrayNode = _jsonFactory.createArrayNode();
                for (JsonNode node : entitiesArrayNode) {
                    Optional<Response> responseOptional = responseList.stream().filter(response -> node.get("tmpId").asLong() == response.getTmpId()).findFirst();
                    if (responseOptional.isPresent()) {
                        Entity entity = responseOptional.get().getEntity();
                        ObjectNode result = _jsonFactory.createObjectNode();
                        result.put("entityType", entity.getType());
                        result.put("id", entity.getId());
                        result.put("context", node.get("context"));
                        arrayNode.add(result);
                    }
                }
                return arrayNode;
            });
        }
        return entitiesField.map(entitiesArray -> {
            ObjectNode payload = objectNode.get("payload").toObjectNode();
            payload.set("entities", entitiesArray);
            return objectNode;
        }).timeout(Duration.ofSeconds(_timeout), Mono.error(new TimeoutException("timeout on entities count: " + entities.size() + " (Did not observe any item or terminal signal within " + Duration.ofSeconds(_timeout).toMillis() + "ms)")));
    });
}
Also used : Response(io.openk9.entity.manager.model.payload.Response) SenderReactor(io.openk9.ingestion.api.SenderReactor) Disposable(reactor.core.Disposable) JsonNode(io.openk9.json.api.JsonNode) LoggerFactory(org.slf4j.LoggerFactory) Tuples(reactor.util.function.Tuples) Tuple2(reactor.util.function.Tuple2) TimeoutException(java.util.concurrent.TimeoutException) EntityRequest(io.openk9.entity.manager.model.payload.EntityRequest) GraphClient(io.openk9.relationship.graph.api.client.GraphClient) ArrayNode(io.openk9.json.api.ArrayNode) JsonFactory(io.openk9.json.api.JsonFactory) ArrayList(java.util.ArrayList) Component(org.osgi.service.component.annotations.Component) Statement(org.neo4j.cypherdsl.core.Statement) Duration(java.time.Duration) OutboundMessageFactory(io.openk9.ingestion.api.OutboundMessageFactory) Schedulers(reactor.core.scheduler.Schedulers) Request(io.openk9.entity.manager.model.payload.Request) Cypher(org.neo4j.cypherdsl.core.Cypher) Activate(org.osgi.service.component.annotations.Activate) Node(org.neo4j.cypherdsl.core.Node) Binding(io.openk9.ingestion.api.Binding) Logger(org.slf4j.Logger) Deactivate(org.osgi.service.component.annotations.Deactivate) Functions(org.neo4j.cypherdsl.core.Functions) RelationRequest(io.openk9.entity.manager.model.payload.RelationRequest) Mono(reactor.core.publisher.Mono) Entity(io.openk9.entity.manager.model.Entity) Collectors(java.util.stream.Collectors) Flux(reactor.core.publisher.Flux) List(java.util.List) Stream(java.util.stream.Stream) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ReceiverReactor(io.openk9.ingestion.api.ReceiverReactor) Modified(org.osgi.service.component.annotations.Modified) Optional(java.util.Optional) ObjectNode(io.openk9.json.api.ObjectNode) Reference(org.osgi.service.component.annotations.Reference) Cypher.literalOf(org.neo4j.cypherdsl.core.Cypher.literalOf) Entity(io.openk9.entity.manager.model.Entity) ObjectNode(io.openk9.json.api.ObjectNode) Optional(java.util.Optional) EntityRequest(io.openk9.entity.manager.model.payload.EntityRequest) Request(io.openk9.entity.manager.model.payload.Request) RelationRequest(io.openk9.entity.manager.model.payload.RelationRequest) JsonNode(io.openk9.json.api.JsonNode) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ArrayList(java.util.ArrayList) List(java.util.List) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ArrayNode(io.openk9.json.api.ArrayNode) TimeoutException(java.util.concurrent.TimeoutException)

Example 2 with Datasource

use of io.openk9.datasource.model.Datasource in project openk9 by smclab.

the class ReindexDatasourceConsumer method activate.

@Activate
void activate() {
    RestHighLevelClient restHighLevelClient = _restHighLevelClientProvider.get();
    _disposable = _datasourceEventConsumer.datasourceUpdateEvents().flatMap(datasource -> _pluginDriverManagerClient.getPluginDriver(datasource.getDriverServiceName()).onErrorResume(throwable -> {
        if (_log.isErrorEnabled()) {
            _log.error(throwable.getMessage());
        }
        return Mono.empty();
    }).map(pluginDriverDTO -> Tuples.of(datasource, datasource.getTenantId() + "-" + pluginDriverDTO.getName() + "-data"))).filterWhen(t2 -> Mono.create(sink -> restHighLevelClient.indices().existsAsync(new GetIndexRequest(t2.getT2()), RequestOptions.DEFAULT, new ReactorActionListener<>(sink)))).flatMap(t2 -> Mono.<GetSettingsResponse>create(sink -> restHighLevelClient.indices().getSettingsAsync(new GetSettingsRequest().indices(t2.getT2()).names("index.creation_date"), RequestOptions.DEFAULT, new ReactorActionListener<>(sink))).map(response -> Tuples.of(t2.getT1(), t2.getT2(), Instant.ofEpochMilli(Long.parseLong(response.getSetting(t2.getT2(), "index.creation_date")))))).log().filter(t3 -> t3.getT1().getLastIngestionDate().isBefore(t3.getT3())).flatMap(t3 -> Mono.<AcknowledgedResponse>create(sink -> restHighLevelClient.indices().deleteAsync(new DeleteIndexRequest(t3.getT2()), RequestOptions.DEFAULT, new ReactorActionListener<>(sink)))).onErrorContinue((throwable, ignore) -> {
        if (_log.isErrorEnabled()) {
            _log.error(throwable.getMessage());
        }
    }).subscribe();
}
Also used : ReactorActionListener(io.openk9.search.client.api.ReactorActionListener) DeleteIndexRequest(org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest) DatasourceEventConsumer(io.openk9.datasource.event.consumer.api.DatasourceEventConsumer) Logger(org.slf4j.Logger) Disposable(reactor.core.Disposable) GetSettingsResponse(org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse) Deactivate(org.osgi.service.component.annotations.Deactivate) LoggerFactory(org.slf4j.LoggerFactory) Tuples(reactor.util.function.Tuples) Mono(reactor.core.publisher.Mono) AcknowledgedResponse(org.elasticsearch.action.support.master.AcknowledgedResponse) RestHighLevelClient(org.elasticsearch.client.RestHighLevelClient) Instant(java.time.Instant) Component(org.osgi.service.component.annotations.Component) RequestOptions(org.elasticsearch.client.RequestOptions) PluginDriverManagerClient(io.openk9.plugin.driver.manager.client.api.PluginDriverManagerClient) Activate(org.osgi.service.component.annotations.Activate) GetSettingsRequest(org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest) Reference(org.osgi.service.component.annotations.Reference) RestHighLevelClientProvider(io.openk9.search.client.api.RestHighLevelClientProvider) GetIndexRequest(org.elasticsearch.client.indices.GetIndexRequest) ReactorActionListener(io.openk9.search.client.api.ReactorActionListener) GetSettingsRequest(org.elasticsearch.action.admin.indices.settings.get.GetSettingsRequest) GetSettingsResponse(org.elasticsearch.action.admin.indices.settings.get.GetSettingsResponse) GetIndexRequest(org.elasticsearch.client.indices.GetIndexRequest) AcknowledgedResponse(org.elasticsearch.action.support.master.AcknowledgedResponse) DeleteIndexRequest(org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest) RestHighLevelClient(org.elasticsearch.client.RestHighLevelClient) Activate(org.osgi.service.component.annotations.Activate)

Example 3 with Datasource

use of io.openk9.datasource.model.Datasource in project openk9 by smclab.

the class SuggestionsHTTPHandler method _getSuggestionsResponse.

private SuggestionsResponse _getSuggestionsResponse(List<Datasource> datasourceList, PluginDriverDTOList pluginDriverDTOList, SearchRequest searchRequest, SearchResponse searchResponse, List<SuggestionCategoryField> fields, Map<String, String[]> entityMap) {
    Aggregations aggregations = searchResponse.getAggregations();
    if (aggregations == null) {
        return SuggestionsResponse.of(List.of(), null);
    }
    Map<String, Aggregation> aggregationMap = aggregations.asMap();
    if (!aggregationMap.containsKey("composite")) {
        return SuggestionsResponse.of(List.of(), null);
    }
    CompositeAggregation compositeAggregation = (CompositeAggregation) aggregationMap.get("composite");
    Map<String, Long> fieldNameCategoryIdMap = fields.stream().collect(Collectors.toMap(SuggestionCategoryField::getFieldName, SuggestionCategoryField::getCategoryId, (a1, a2) -> a2));
    Long datasourceIdCategoryId = fieldNameCategoryIdMap.getOrDefault("datasourceId", 1L);
    Long entityIdCategoryId = fieldNameCategoryIdMap.getOrDefault("entities.id", 2L);
    Long entitiesContextCategoryId = fieldNameCategoryIdMap.getOrDefault("entities.context", entityIdCategoryId);
    Long documentTypesCategoryId = fieldNameCategoryIdMap.getOrDefault("documentTypes", 4L);
    List<? extends CompositeAggregation.Bucket> buckets = compositeAggregation.getBuckets();
    LinkedList<Suggestions> suggestions = new LinkedList<>();
    String suggestKeyword = searchRequest.getSuggestKeyword();
    BiConsumer<String, Suggestions> addSuggestions;
    if (suggestKeyword != null) {
        addSuggestions = (key, sugg) -> {
            if (!suggestions.contains(sugg)) {
                if (key.contains(suggestKeyword)) {
                    suggestions.addFirst(sugg);
                }
            }
        };
    } else {
        addSuggestions = (key, sugg) -> {
            if (!suggestions.contains(sugg)) {
                suggestions.add(sugg);
            }
        };
    }
    for (CompositeAggregation.Bucket bucket : buckets) {
        Map<String, Object> keys = new HashMap<>(bucket.getKey());
        for (Map.Entry<String, Object> entry : keys.entrySet()) {
            String key = entry.getKey();
            String value = (String) entry.getValue();
            if (value == null) {
                continue;
            }
            switch(key) {
                case "datasourceId":
                    long datasourceIdL = Long.parseLong(value);
                    _datasource(datasourceList, pluginDriverDTOList, datasourceIdCategoryId, addSuggestions, datasourceIdL);
                    break;
                case "entities.context":
                    break;
                case "entities.id":
                    String[] typeName = entityMap.get(value);
                    if (typeName != null) {
                        String type = typeName[0];
                        String name = typeName[1];
                        String entitiesContext = (String) keys.get("entities.context");
                        if (entitiesContext != null) {
                            addSuggestions.accept(name, Suggestions.entity(value, entitiesContextCategoryId, type, name, entitiesContext));
                        } else {
                            addSuggestions.accept(name, Suggestions.entity(value, entityIdCategoryId, type, name));
                        }
                    }
                    break;
                case "documentTypes":
                    addSuggestions.accept(value, Suggestions.docType(value, documentTypesCategoryId));
                    break;
                default:
                    Long textCategoryId = fieldNameCategoryIdMap.getOrDefault(key, 5L);
                    addSuggestions.accept(value, Suggestions.text(value, textCategoryId, key));
            }
        }
    }
    Map<String, Object> map = compositeAggregation.afterKey();
    String afterKey = null;
    int[] range = searchRequest.getRange();
    if (map != null) {
        afterKey = _jsonFactory.toJson(map);
        afterKey = Base64.getEncoder().encodeToString(afterKey.getBytes(StandardCharsets.UTF_8));
    }
    return SuggestionsResponse.of(suggestions, afterKey);
}
Also used : PluginDriverDTO(io.openk9.plugin.driver.manager.model.PluginDriverDTO) HttpUtil(io.openk9.http.util.HttpUtil) PluginDriverDTOList(io.openk9.plugin.driver.manager.model.PluginDriverDTOList) LoggerFactory(org.slf4j.LoggerFactory) SuggestionsResponse(io.openk9.search.query.internal.response.SuggestionsResponse) QueryParser(io.openk9.search.api.query.QueryParser) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) SearchToken(io.openk9.search.api.query.SearchToken) JsonFactory(io.openk9.json.api.JsonFactory) Map(java.util.Map) SearchResponse(org.elasticsearch.action.search.SearchResponse) PluginDriverManagerClient(io.openk9.plugin.driver.manager.client.api.PluginDriverManagerClient) SearchHit(org.elasticsearch.search.SearchHit) Tenant(io.openk9.model.Tenant) Aggregations(org.elasticsearch.search.aggregations.Aggregations) DatasourceClient(io.openk9.datasource.client.api.DatasourceClient) ReferencePolicy(org.osgi.service.component.annotations.ReferencePolicy) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) Objects(java.util.Objects) HttpServerRequest(reactor.netty.http.server.HttpServerRequest) Base64(java.util.Base64) List(java.util.List) ReferencePolicyOption(org.osgi.service.component.annotations.ReferencePolicyOption) SuggestionCategoryField(io.openk9.model.SuggestionCategoryField) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) HttpServerRoutes(reactor.netty.http.server.HttpServerRoutes) HttpServerResponse(reactor.netty.http.server.HttpServerResponse) Search(io.openk9.search.client.api.Search) Tuple2(reactor.util.function.Tuple2) HashMap(java.util.HashMap) HttpResponseWriter(io.openk9.http.util.HttpResponseWriter) Function(java.util.function.Function) Component(org.osgi.service.component.annotations.Component) CompositeAggregation(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation) CompositeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) BiConsumer(java.util.function.BiConsumer) Aggregation(org.elasticsearch.search.aggregations.Aggregation) Activate(org.osgi.service.component.annotations.Activate) LinkedList(java.util.LinkedList) RouterHandler(io.openk9.http.web.RouterHandler) CompositeValuesSourceBuilder(org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder) Logger(org.slf4j.Logger) Datasource(io.openk9.model.Datasource) Suggestions(io.openk9.search.query.internal.response.suggestions.Suggestions) AggregationBuilders(org.elasticsearch.search.aggregations.AggregationBuilders) Mono(reactor.core.publisher.Mono) TermsValuesSourceBuilder(org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder) ReferenceCardinality(org.osgi.service.component.annotations.ReferenceCardinality) SearchRequest(io.openk9.search.api.query.SearchRequest) SearchTokenizer(io.openk9.search.api.query.SearchTokenizer) Modified(org.osgi.service.component.annotations.Modified) Reference(org.osgi.service.component.annotations.Reference) CompositeAggregation(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation) HashMap(java.util.HashMap) Aggregations(org.elasticsearch.search.aggregations.Aggregations) LinkedList(java.util.LinkedList) CompositeAggregation(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation) Aggregation(org.elasticsearch.search.aggregations.Aggregation) Suggestions(io.openk9.search.query.internal.response.suggestions.Suggestions) Map(java.util.Map) HashMap(java.util.HashMap)

Example 4 with Datasource

use of io.openk9.datasource.model.Datasource in project openk9 by smclab.

the class SupportedDatasourcesEndpoint method _mapToResponse.

private SupportedDatasourcesResponse _mapToResponse(Tuple2<Datasource, PluginDriverDTO> t2) {
    Datasource t1 = t2.getT1();
    PluginDriverDTO pluginDriver = t2.getT2();
    return SupportedDatasourcesResponse.of(t1.getName(), t1.getActive(), pluginDriver.getDocumentTypes().stream().map(documentTypeDTO -> SupportedDatasourcesDocumentType.of(documentTypeDTO.getName(), documentTypeDTO.getIcon())).collect(Collectors.toList()), SupportedDatasourcesDocumentType.of(pluginDriver.getDefaultDocumentType().getName(), pluginDriver.getDefaultDocumentType().getIcon()));
}
Also used : Datasource(io.openk9.model.Datasource) PluginDriverDTO(io.openk9.plugin.driver.manager.model.PluginDriverDTO)

Example 5 with Datasource

use of io.openk9.datasource.model.Datasource in project openk9 by smclab.

the class DocumentTypesEndpoint method _mapToResponse.

private Object _mapToResponse(List<Tuple2<Datasource, PluginDriverDTO>> list) {
    Map<String, Collection<String>> response = new HashMap<>();
    for (Tuple2<Datasource, PluginDriverDTO> t2 : list) {
        PluginDriverDTO pluginDriver = t2.getT2();
        for (DocumentTypeDTO documentType : pluginDriver.getDocumentTypes()) {
            String name = documentType.getName();
            List<SearchKeywordDTO> searchKeywords = documentType.getSearchKeywords();
            List<String> keywords = searchKeywords.stream().map(SearchKeywordDTO::getKeyword).collect(Collectors.toList());
            Collection<String> prevKeywords = response.get(name);
            Set<String> combine = new HashSet<>(keywords);
            if (prevKeywords != null) {
                combine.addAll(prevKeywords);
            }
            response.put(name, combine);
        }
    }
    return response;
}
Also used : Datasource(io.openk9.model.Datasource) HashMap(java.util.HashMap) SearchKeywordDTO(io.openk9.plugin.driver.manager.model.SearchKeywordDTO) Collection(java.util.Collection) DocumentTypeDTO(io.openk9.plugin.driver.manager.model.DocumentTypeDTO) PluginDriverDTO(io.openk9.plugin.driver.manager.model.PluginDriverDTO) HashSet(java.util.HashSet)

Aggregations

Datasource (io.openk9.model.Datasource)12 Mono (reactor.core.publisher.Mono)11 List (java.util.List)9 Activate (org.osgi.service.component.annotations.Activate)9 Component (org.osgi.service.component.annotations.Component)9 Logger (org.slf4j.Logger)9 LoggerFactory (org.slf4j.LoggerFactory)9 JsonFactory (io.openk9.json.api.JsonFactory)8 PluginDriverManagerClient (io.openk9.plugin.driver.manager.client.api.PluginDriverManagerClient)8 Map (java.util.Map)8 Reference (org.osgi.service.component.annotations.Reference)8 PluginDriverDTO (io.openk9.plugin.driver.manager.model.PluginDriverDTO)7 Collectors (java.util.stream.Collectors)7 DatasourceClient (io.openk9.datasource.client.api.DatasourceClient)6 RouterHandler (io.openk9.http.web.RouterHandler)6 Tenant (io.openk9.model.Tenant)6 HashMap (java.util.HashMap)6 BoolQueryBuilder (org.elasticsearch.index.query.BoolQueryBuilder)6 QueryBuilders (org.elasticsearch.index.query.QueryBuilders)6 HttpResponseWriter (io.openk9.http.util.HttpResponseWriter)5