Search in sources :

Example 1 with Tenant

use of io.openk9.api.aggregator.model.Tenant in project openk9 by smclab.

the class GetOrAddEntitiesConsumer method apply.

public Mono<ObjectNode> apply(ObjectNode objectNode) {
    return Mono.defer(() -> {
        ObjectNode datasourceContextJson = objectNode.get("datasourceContext").toObjectNode();
        long datasourceId = datasourceContextJson.get("datasource").get("datasourceId").asLong();
        long tenantId = datasourceContextJson.get("tenant").get("tenantId").asLong();
        JsonNode entities = objectNode.remove("entities");
        Mono<ArrayNode> entitiesField;
        if (entities.size() == 0) {
            entitiesField = Mono.just(_jsonFactory.createArrayNode());
        } else {
            ObjectNode responseJson = _jsonFactory.createObjectNode();
            responseJson.put("entities", entities);
            responseJson.put("tenantId", tenantId);
            responseJson.put("datasourceId", datasourceId);
            Request request = _jsonFactory.fromJson(responseJson.toString(), Request.class);
            List<RequestContext> requestContextList = request.getEntities().stream().map(entityRequest -> RequestContext.builder().current(entityRequest).tenantId(request.getTenantId()).datasourceId(request.getDatasourceId()).rest(request.getEntities().stream().filter(er -> er != entityRequest).collect(Collectors.toList())).build()).collect(Collectors.toList());
            Mono<List<EntityContext>> disambiguateListMono = GetOrAddEntities.stopWatch("disambiguate-all-entities", Flux.fromIterable(requestContextList).flatMap(requestContext -> GetOrAddEntities.stopWatch("disambiguate-" + requestContext.getCurrent().getName(), Mono.<EntityContext>create(fluxSink -> _startDisambiguation.disambiguate(requestContext, fluxSink)))).collectList());
            Mono<ResponseList> writeRelations = disambiguateListMono.flatMap(entityContexts -> GetOrAddEntities.stopWatch("write-relations", writeRelations(entityContexts)));
            Mono<ResponseList> responseListWrapper = _transactional ? _graphClient.makeTransactional(writeRelations) : writeRelations;
            entitiesField = responseListWrapper.map(responseListDTO -> {
                List<Response> responseList = responseListDTO.getResponse();
                ArrayNode entitiesArrayNode = entities.toArrayNode();
                ArrayNode arrayNode = _jsonFactory.createArrayNode();
                for (JsonNode node : entitiesArrayNode) {
                    Optional<Response> responseOptional = responseList.stream().filter(response -> node.get("tmpId").asLong() == response.getTmpId()).findFirst();
                    if (responseOptional.isPresent()) {
                        Entity entity = responseOptional.get().getEntity();
                        ObjectNode result = _jsonFactory.createObjectNode();
                        result.put("entityType", entity.getType());
                        result.put("id", entity.getId());
                        result.put("context", node.get("context"));
                        arrayNode.add(result);
                    }
                }
                return arrayNode;
            });
        }
        return entitiesField.map(entitiesArray -> {
            ObjectNode payload = objectNode.get("payload").toObjectNode();
            payload.set("entities", entitiesArray);
            return objectNode;
        }).timeout(Duration.ofSeconds(_timeout), Mono.error(new TimeoutException("timeout on entities count: " + entities.size() + " (Did not observe any item or terminal signal within " + Duration.ofSeconds(_timeout).toMillis() + "ms)")));
    });
}
Also used : Response(io.openk9.entity.manager.model.payload.Response) SenderReactor(io.openk9.ingestion.api.SenderReactor) Disposable(reactor.core.Disposable) JsonNode(io.openk9.json.api.JsonNode) LoggerFactory(org.slf4j.LoggerFactory) Tuples(reactor.util.function.Tuples) Tuple2(reactor.util.function.Tuple2) TimeoutException(java.util.concurrent.TimeoutException) EntityRequest(io.openk9.entity.manager.model.payload.EntityRequest) GraphClient(io.openk9.relationship.graph.api.client.GraphClient) ArrayNode(io.openk9.json.api.ArrayNode) JsonFactory(io.openk9.json.api.JsonFactory) ArrayList(java.util.ArrayList) Component(org.osgi.service.component.annotations.Component) Statement(org.neo4j.cypherdsl.core.Statement) Duration(java.time.Duration) OutboundMessageFactory(io.openk9.ingestion.api.OutboundMessageFactory) Schedulers(reactor.core.scheduler.Schedulers) Request(io.openk9.entity.manager.model.payload.Request) Cypher(org.neo4j.cypherdsl.core.Cypher) Activate(org.osgi.service.component.annotations.Activate) Node(org.neo4j.cypherdsl.core.Node) Binding(io.openk9.ingestion.api.Binding) Logger(org.slf4j.Logger) Deactivate(org.osgi.service.component.annotations.Deactivate) Functions(org.neo4j.cypherdsl.core.Functions) RelationRequest(io.openk9.entity.manager.model.payload.RelationRequest) Mono(reactor.core.publisher.Mono) Entity(io.openk9.entity.manager.model.Entity) Collectors(java.util.stream.Collectors) Flux(reactor.core.publisher.Flux) List(java.util.List) Stream(java.util.stream.Stream) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ReceiverReactor(io.openk9.ingestion.api.ReceiverReactor) Modified(org.osgi.service.component.annotations.Modified) Optional(java.util.Optional) ObjectNode(io.openk9.json.api.ObjectNode) Reference(org.osgi.service.component.annotations.Reference) Cypher.literalOf(org.neo4j.cypherdsl.core.Cypher.literalOf) Entity(io.openk9.entity.manager.model.Entity) ObjectNode(io.openk9.json.api.ObjectNode) Optional(java.util.Optional) EntityRequest(io.openk9.entity.manager.model.payload.EntityRequest) Request(io.openk9.entity.manager.model.payload.Request) RelationRequest(io.openk9.entity.manager.model.payload.RelationRequest) JsonNode(io.openk9.json.api.JsonNode) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ArrayList(java.util.ArrayList) List(java.util.List) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ArrayNode(io.openk9.json.api.ArrayNode) TimeoutException(java.util.concurrent.TimeoutException)

Example 2 with Tenant

use of io.openk9.api.aggregator.model.Tenant in project openk9 by smclab.

the class AuthResource method logout.

@PermitAll
@Path("/v1/auth/logout")
@POST
public Uni<byte[]> logout(@Context HttpServerRequest context, RefreshToken request) {
    String host = context.host();
    Tenant tenant = _findTenant(host);
    String realmName = tenant.getRealmName();
    String clientSecret = tenant.getClientSecret();
    String clientId = tenant.getClientId();
    String refreshToken = request.getRefreshToken();
    if (refreshToken == null) {
        return Uni.createFrom().failure(() -> new HttpException(400, "required refreshToken"));
    }
    if (clientSecret != null) {
        return _authClient.logout(realmName, clientId, clientSecret, refreshToken);
    } else {
        return _authClient.logout(realmName, clientId, refreshToken);
    }
}
Also used : Tenant(io.openk9.api.aggregator.model.Tenant) HttpException(io.vertx.ext.web.handler.HttpException) Path(javax.ws.rs.Path) POST(javax.ws.rs.POST) PermitAll(javax.annotation.security.PermitAll)

Example 3 with Tenant

use of io.openk9.api.aggregator.model.Tenant in project openk9 by smclab.

the class AuthResource method refreshToken.

@PermitAll
@Path("/v1/auth/refresh")
@POST
public Uni<LoginResponseDTO> refreshToken(@Context HttpServerRequest context, RefreshToken request) {
    String host = context.host();
    Tenant tenant = _findTenant(host);
    String realmName = tenant.getRealmName();
    String clientSecret = tenant.getClientSecret();
    String clientId = tenant.getClientId();
    String refreshToken = request.getRefreshToken();
    if (refreshToken == null) {
        return Uni.createFrom().failure(() -> new HttpException(400, "required refreshToken"));
    }
    if (clientSecret != null) {
        return _authClient.refresh(realmName, clientId, clientSecret, refreshToken, "refresh_token");
    } else {
        return _authClient.refresh(realmName, clientId, refreshToken, "refresh_token");
    }
}
Also used : Tenant(io.openk9.api.aggregator.model.Tenant) HttpException(io.vertx.ext.web.handler.HttpException) Path(javax.ws.rs.Path) POST(javax.ws.rs.POST) PermitAll(javax.annotation.security.PermitAll)

Example 4 with Tenant

use of io.openk9.api.aggregator.model.Tenant in project openk9 by smclab.

the class DatasourceProcessor method process.

@Incoming("ingestion")
@Outgoing("ingestion-datasource")
public Uni<IngestionDatasourcePayload> process(Object obj) {
    JsonObject jsonObject = obj instanceof JsonObject ? (JsonObject) obj : new JsonObject(new String((byte[]) obj));
    long datasourceId = jsonObject.getLong("datasourceId");
    Uni<Datasource> datasourceUni = Datasource.findById(datasourceId);
    return Panache.withTransaction(() -> datasourceUni.flatMap(datasource -> EnrichPipeline.findByDatasourceId(datasource.getDatasourceId()).onItem().ifNull().continueWith(EnrichPipeline::new).flatMap(enrichPipeline -> {
        Uni<List<EnrichItem>> enrichItemUni;
        if (enrichPipeline.getEnrichPipelineId() != null) {
            enrichItemUni = EnrichItem.findByEnrichPipelineId(enrichPipeline.getEnrichPipelineId()).onItem().ifNull().continueWith(List::of);
        } else {
            enrichItemUni = Uni.createFrom().item(List.of());
        }
        return Uni.combine().all().unis(Tenant.findById(datasource.getTenantId()), enrichItemUni).combinedWith((tenantObj, enrichItemList) -> {
            Tenant tenant = (Tenant) tenantObj;
            IngestionPayload ingestionPayload = jsonObject.mapTo(IngestionPayload.class);
            ingestionPayload.setTenantId(tenant.getTenantId());
            DatasourceContext datasourceContext = DatasourceContext.of(datasource, tenant, enrichPipeline, enrichItemList);
            return IngestionDatasourcePayload.of(ingestionPayload, datasourceContext);
        });
    })).eventually(() -> Datasource.<Datasource>findById(datasourceId).flatMap(datasource -> {
        datasource.setLastIngestionDate(Instant.ofEpochMilli(jsonObject.getLong("parsingDate")));
        return datasource.persist();
    })));
}
Also used : Datasource(io.openk9.datasource.model.Datasource) Outgoing(org.eclipse.microprofile.reactive.messaging.Outgoing) Datasource(io.openk9.datasource.model.Datasource) EnrichItem(io.openk9.datasource.model.EnrichItem) Tenant(io.openk9.datasource.model.Tenant) Instant(java.time.Instant) Uni(io.smallrye.mutiny.Uni) DatasourceContext(io.openk9.datasource.processor.payload.DatasourceContext) List(java.util.List) IngestionDatasourcePayload(io.openk9.datasource.processor.payload.IngestionDatasourcePayload) IngestionPayload(io.openk9.datasource.processor.payload.IngestionPayload) EnrichPipeline(io.openk9.datasource.model.EnrichPipeline) JsonObject(io.vertx.core.json.JsonObject) ApplicationScoped(javax.enterprise.context.ApplicationScoped) Panache(io.quarkus.hibernate.reactive.panache.Panache) Incoming(org.eclipse.microprofile.reactive.messaging.Incoming) JsonObject(io.vertx.core.json.JsonObject) EnrichItem(io.openk9.datasource.model.EnrichItem) Uni(io.smallrye.mutiny.Uni) Tenant(io.openk9.datasource.model.Tenant) IngestionPayload(io.openk9.datasource.processor.payload.IngestionPayload) DatasourceContext(io.openk9.datasource.processor.payload.DatasourceContext) EnrichPipeline(io.openk9.datasource.model.EnrichPipeline) List(java.util.List) Incoming(org.eclipse.microprofile.reactive.messaging.Incoming) Outgoing(org.eclipse.microprofile.reactive.messaging.Outgoing)

Example 5 with Tenant

use of io.openk9.api.aggregator.model.Tenant in project openk9 by smclab.

the class SuggestionsHTTPHandler method searchHitToResponseMono.

@Override
protected Mono<Object> searchHitToResponseMono(Tenant tenant, List<Datasource> datasourceList, PluginDriverDTOList pluginDriverDTOList, HttpServerRequest httpServerRequest, SearchRequest searchRequest, SearchResponse searchResponse) {
    Long suggestionCategoryId = searchRequest.getSuggestionCategoryId();
    Mono<List<SuggestionCategoryField>> suggestionCategoryFields;
    if (suggestionCategoryId == null) {
        suggestionCategoryFields = _datasourceClient.findSuggestionCategoryFieldsByTenantId(tenant.getTenantId());
    } else {
        suggestionCategoryFields = _datasourceClient.findSuggestionCategoryFieldsByTenantIdAndCategoryId(tenant.getTenantId(), suggestionCategoryId);
    }
    if (_enableEntityAggregation) {
        return _search.search(factory -> {
            org.elasticsearch.action.search.SearchRequest searchRequestEntity = factory.createSearchRequestEntity(tenant.getTenantId());
            Aggregations aggregations = searchResponse.getAggregations();
            CompositeAggregation compositeAggregation = aggregations.get("composite");
            List<? extends CompositeAggregation.Bucket> buckets = compositeAggregation.getBuckets();
            BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
            buckets.stream().map(bucket -> (String) bucket.getKey().get("entities.id")).filter(Objects::nonNull).distinct().forEach(entityId -> boolQueryBuilder.should(QueryBuilders.matchQuery("id", entityId)));
            if (_log.isDebugEnabled()) {
                _log.debug("entities query: " + boolQueryBuilder);
            }
            SearchSourceBuilder ssb = new SearchSourceBuilder();
            ssb.query(boolQueryBuilder);
            ssb.size(1000);
            ssb.fetchSource(new String[] { "name", "id", "type" }, null);
            return searchRequestEntity.source(ssb);
        }).flatMap(entityResponse -> suggestionCategoryFields.map(fields -> {
            Map<String, String[]> entityMap = new HashMap<>();
            for (SearchHit hit : entityResponse.getHits()) {
                Map<String, Object> sourceAsMap = hit.getSourceAsMap();
                String name = (String) sourceAsMap.get("name");
                String type = (String) sourceAsMap.get("type");
                String entityId = (String) sourceAsMap.get("id");
                entityMap.put(entityId, new String[] { type, name });
            }
            return _getSuggestionsResponse(datasourceList, pluginDriverDTOList, searchRequest, searchResponse, fields, entityMap);
        }));
    } else {
        return suggestionCategoryFields.map(fields -> _getSuggestionsResponse(datasourceList, pluginDriverDTOList, searchRequest, searchResponse, fields, Map.of()));
    }
}
Also used : PluginDriverDTO(io.openk9.plugin.driver.manager.model.PluginDriverDTO) HttpUtil(io.openk9.http.util.HttpUtil) PluginDriverDTOList(io.openk9.plugin.driver.manager.model.PluginDriverDTOList) LoggerFactory(org.slf4j.LoggerFactory) SuggestionsResponse(io.openk9.search.query.internal.response.SuggestionsResponse) QueryParser(io.openk9.search.api.query.QueryParser) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) SearchToken(io.openk9.search.api.query.SearchToken) JsonFactory(io.openk9.json.api.JsonFactory) Map(java.util.Map) SearchResponse(org.elasticsearch.action.search.SearchResponse) PluginDriverManagerClient(io.openk9.plugin.driver.manager.client.api.PluginDriverManagerClient) SearchHit(org.elasticsearch.search.SearchHit) Tenant(io.openk9.model.Tenant) Aggregations(org.elasticsearch.search.aggregations.Aggregations) DatasourceClient(io.openk9.datasource.client.api.DatasourceClient) ReferencePolicy(org.osgi.service.component.annotations.ReferencePolicy) Collectors(java.util.stream.Collectors) StandardCharsets(java.nio.charset.StandardCharsets) Objects(java.util.Objects) HttpServerRequest(reactor.netty.http.server.HttpServerRequest) Base64(java.util.Base64) List(java.util.List) ReferencePolicyOption(org.osgi.service.component.annotations.ReferencePolicyOption) SuggestionCategoryField(io.openk9.model.SuggestionCategoryField) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) HttpServerRoutes(reactor.netty.http.server.HttpServerRoutes) HttpServerResponse(reactor.netty.http.server.HttpServerResponse) Search(io.openk9.search.client.api.Search) Tuple2(reactor.util.function.Tuple2) HashMap(java.util.HashMap) HttpResponseWriter(io.openk9.http.util.HttpResponseWriter) Function(java.util.function.Function) Component(org.osgi.service.component.annotations.Component) CompositeAggregation(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation) CompositeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregationBuilder) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) BiConsumer(java.util.function.BiConsumer) Aggregation(org.elasticsearch.search.aggregations.Aggregation) Activate(org.osgi.service.component.annotations.Activate) LinkedList(java.util.LinkedList) RouterHandler(io.openk9.http.web.RouterHandler) CompositeValuesSourceBuilder(org.elasticsearch.search.aggregations.bucket.composite.CompositeValuesSourceBuilder) Logger(org.slf4j.Logger) Datasource(io.openk9.model.Datasource) Suggestions(io.openk9.search.query.internal.response.suggestions.Suggestions) AggregationBuilders(org.elasticsearch.search.aggregations.AggregationBuilders) Mono(reactor.core.publisher.Mono) TermsValuesSourceBuilder(org.elasticsearch.search.aggregations.bucket.composite.TermsValuesSourceBuilder) ReferenceCardinality(org.osgi.service.component.annotations.ReferenceCardinality) SearchRequest(io.openk9.search.api.query.SearchRequest) SearchTokenizer(io.openk9.search.api.query.SearchTokenizer) Modified(org.osgi.service.component.annotations.Modified) Reference(org.osgi.service.component.annotations.Reference) SearchRequest(io.openk9.search.api.query.SearchRequest) CompositeAggregation(org.elasticsearch.search.aggregations.bucket.composite.CompositeAggregation) SearchHit(org.elasticsearch.search.SearchHit) Aggregations(org.elasticsearch.search.aggregations.Aggregations) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) PluginDriverDTOList(io.openk9.plugin.driver.manager.model.PluginDriverDTOList) List(java.util.List) LinkedList(java.util.LinkedList) Map(java.util.Map) HashMap(java.util.HashMap)

Aggregations

Tenant (io.openk9.api.aggregator.model.Tenant)6 List (java.util.List)6 Activate (org.osgi.service.component.annotations.Activate)6 JsonFactory (io.openk9.json.api.JsonFactory)5 Datasource (io.openk9.model.Datasource)5 Tenant (io.openk9.model.Tenant)5 Logger (org.slf4j.Logger)5 LoggerFactory (org.slf4j.LoggerFactory)5 DatasourceClient (io.openk9.datasource.client.api.DatasourceClient)4 RouterHandler (io.openk9.http.web.RouterHandler)4 PluginDriverManagerClient (io.openk9.plugin.driver.manager.client.api.PluginDriverManagerClient)4 PluginDriverDTO (io.openk9.plugin.driver.manager.model.PluginDriverDTO)4 QueryParser (io.openk9.search.api.query.QueryParser)4 Search (io.openk9.search.client.api.Search)4 HashMap (java.util.HashMap)4 Map (java.util.Map)4 Collectors (java.util.stream.Collectors)4 BoolQueryBuilder (org.elasticsearch.index.query.BoolQueryBuilder)4 QueryBuilders (org.elasticsearch.index.query.QueryBuilders)4 SearchHit (org.elasticsearch.search.SearchHit)4