Search in sources :

Example 1 with DatasourceContext

use of io.openk9.model.DatasourceContext in project openk9 by smclab.

the class GetOrAddEntitiesConsumer method apply.

public Mono<ObjectNode> apply(ObjectNode objectNode) {
    return Mono.defer(() -> {
        ObjectNode datasourceContextJson = objectNode.get("datasourceContext").toObjectNode();
        long datasourceId = datasourceContextJson.get("datasource").get("datasourceId").asLong();
        long tenantId = datasourceContextJson.get("tenant").get("tenantId").asLong();
        JsonNode entities = objectNode.remove("entities");
        Mono<ArrayNode> entitiesField;
        if (entities.size() == 0) {
            entitiesField = Mono.just(_jsonFactory.createArrayNode());
        } else {
            ObjectNode responseJson = _jsonFactory.createObjectNode();
            responseJson.put("entities", entities);
            responseJson.put("tenantId", tenantId);
            responseJson.put("datasourceId", datasourceId);
            Request request = _jsonFactory.fromJson(responseJson.toString(), Request.class);
            List<RequestContext> requestContextList = request.getEntities().stream().map(entityRequest -> RequestContext.builder().current(entityRequest).tenantId(request.getTenantId()).datasourceId(request.getDatasourceId()).rest(request.getEntities().stream().filter(er -> er != entityRequest).collect(Collectors.toList())).build()).collect(Collectors.toList());
            Mono<List<EntityContext>> disambiguateListMono = GetOrAddEntities.stopWatch("disambiguate-all-entities", Flux.fromIterable(requestContextList).flatMap(requestContext -> GetOrAddEntities.stopWatch("disambiguate-" + requestContext.getCurrent().getName(), Mono.<EntityContext>create(fluxSink -> _startDisambiguation.disambiguate(requestContext, fluxSink)))).collectList());
            Mono<ResponseList> writeRelations = disambiguateListMono.flatMap(entityContexts -> GetOrAddEntities.stopWatch("write-relations", writeRelations(entityContexts)));
            Mono<ResponseList> responseListWrapper = _transactional ? _graphClient.makeTransactional(writeRelations) : writeRelations;
            entitiesField = responseListWrapper.map(responseListDTO -> {
                List<Response> responseList = responseListDTO.getResponse();
                ArrayNode entitiesArrayNode = entities.toArrayNode();
                ArrayNode arrayNode = _jsonFactory.createArrayNode();
                for (JsonNode node : entitiesArrayNode) {
                    Optional<Response> responseOptional = responseList.stream().filter(response -> node.get("tmpId").asLong() == response.getTmpId()).findFirst();
                    if (responseOptional.isPresent()) {
                        Entity entity = responseOptional.get().getEntity();
                        ObjectNode result = _jsonFactory.createObjectNode();
                        result.put("entityType", entity.getType());
                        result.put("id", entity.getId());
                        result.put("context", node.get("context"));
                        arrayNode.add(result);
                    }
                }
                return arrayNode;
            });
        }
        return entitiesField.map(entitiesArray -> {
            ObjectNode payload = objectNode.get("payload").toObjectNode();
            payload.set("entities", entitiesArray);
            return objectNode;
        }).timeout(Duration.ofSeconds(_timeout), Mono.error(new TimeoutException("timeout on entities count: " + entities.size() + " (Did not observe any item or terminal signal within " + Duration.ofSeconds(_timeout).toMillis() + "ms)")));
    });
}
Also used : Response(io.openk9.entity.manager.model.payload.Response) SenderReactor(io.openk9.ingestion.api.SenderReactor) Disposable(reactor.core.Disposable) JsonNode(io.openk9.json.api.JsonNode) LoggerFactory(org.slf4j.LoggerFactory) Tuples(reactor.util.function.Tuples) Tuple2(reactor.util.function.Tuple2) TimeoutException(java.util.concurrent.TimeoutException) EntityRequest(io.openk9.entity.manager.model.payload.EntityRequest) GraphClient(io.openk9.relationship.graph.api.client.GraphClient) ArrayNode(io.openk9.json.api.ArrayNode) JsonFactory(io.openk9.json.api.JsonFactory) ArrayList(java.util.ArrayList) Component(org.osgi.service.component.annotations.Component) Statement(org.neo4j.cypherdsl.core.Statement) Duration(java.time.Duration) OutboundMessageFactory(io.openk9.ingestion.api.OutboundMessageFactory) Schedulers(reactor.core.scheduler.Schedulers) Request(io.openk9.entity.manager.model.payload.Request) Cypher(org.neo4j.cypherdsl.core.Cypher) Activate(org.osgi.service.component.annotations.Activate) Node(org.neo4j.cypherdsl.core.Node) Binding(io.openk9.ingestion.api.Binding) Logger(org.slf4j.Logger) Deactivate(org.osgi.service.component.annotations.Deactivate) Functions(org.neo4j.cypherdsl.core.Functions) RelationRequest(io.openk9.entity.manager.model.payload.RelationRequest) Mono(reactor.core.publisher.Mono) Entity(io.openk9.entity.manager.model.Entity) Collectors(java.util.stream.Collectors) Flux(reactor.core.publisher.Flux) List(java.util.List) Stream(java.util.stream.Stream) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ReceiverReactor(io.openk9.ingestion.api.ReceiverReactor) Modified(org.osgi.service.component.annotations.Modified) Optional(java.util.Optional) ObjectNode(io.openk9.json.api.ObjectNode) Reference(org.osgi.service.component.annotations.Reference) Cypher.literalOf(org.neo4j.cypherdsl.core.Cypher.literalOf) Entity(io.openk9.entity.manager.model.Entity) ObjectNode(io.openk9.json.api.ObjectNode) Optional(java.util.Optional) EntityRequest(io.openk9.entity.manager.model.payload.EntityRequest) Request(io.openk9.entity.manager.model.payload.Request) RelationRequest(io.openk9.entity.manager.model.payload.RelationRequest) JsonNode(io.openk9.json.api.JsonNode) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ArrayList(java.util.ArrayList) List(java.util.List) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ArrayNode(io.openk9.json.api.ArrayNode) TimeoutException(java.util.concurrent.TimeoutException)

Example 2 with DatasourceContext

use of io.openk9.model.DatasourceContext in project openk9 by smclab.

the class BaseNerEnrichProcessor method prepareRequestRawContent.

protected ObjectNode prepareRequestRawContent(ObjectNode objectNode, ObjectNode datasourceConfiguration, DatasourceContext context, PluginDriverDTO pluginDriverDTO) {
    JsonNode entitiesNode = datasourceConfiguration.get(entitiesField());
    JsonNode relationsNode = datasourceConfiguration.get(relationsField());
    JsonNode rawContentNode = objectNode.get(Constants.RAW_CONTENT);
    JsonNode confidenceNode = datasourceConfiguration.get(Constants.CONFIDENCE);
    ObjectNode request = _jsonFactory.createObjectNode();
    request.put(entitiesField(), entitiesNode);
    request.put(relationsField(), relationsNode);
    request.put(Constants.CONFIDENCE, confidenceNode);
    request.put(Constants.CONTENT, rawContentNode);
    JsonNode typeNode = objectNode.get(Constants.TYPE);
    ObjectNode datasourcePayload = _jsonFactory.createObjectNode();
    if (typeNode != null && typeNode.isArray()) {
        ArrayNode types = typeNode.toArrayNode();
        for (JsonNode typeJsonNode : types) {
            String type = typeJsonNode.asText();
            datasourcePayload.put(type, objectNode.get(type));
        }
    }
    request.put(Constants.DATASOURCE_PAYLOAD, datasourcePayload);
    request.put(Constants.TENANT_ID, context.getTenant().getTenantId());
    request.put(Constants.DATASOURCE_ID, context.getDatasource().getDatasourceId());
    request.put(Constants.CONTENT_ID, objectNode.get(Constants.CONTENT_ID));
    return request;
}
Also used : ObjectNode(io.openk9.json.api.ObjectNode) JsonNode(io.openk9.json.api.JsonNode) ArrayNode(io.openk9.json.api.ArrayNode)

Example 3 with DatasourceContext

use of io.openk9.model.DatasourceContext in project openk9 by smclab.

the class JsEnrichProcessor method process.

@Override
public Mono<ObjectNode> process(ObjectNode objectNode, DatasourceContext context, EnrichItem enrichItem, PluginDriverDTO pluginDriverName) {
    return Mono.defer(() -> {
        JsonNode datasourceConfiguration = _jsonFactory.fromJsonToJsonNode(enrichItem.getJsonConfig());
        if (!datasourceConfiguration.isObject()) {
            return Mono.error(new RuntimeException("jsonConfig must be an instance of ObjectNode " + datasourceConfiguration.toString()));
        }
        ObjectNode request = prepareRequestRawContent(objectNode, datasourceConfiguration.toObjectNode(), context, pluginDriverName);
        return Mono.from(_httpClient.request(getMethod(), getPath(), request.toString(), getHeaders())).map(_jsonFactory::fromJsonToJsonNode).map(JsonNode::toObjectNode).map(objectNode::merge);
    });
}
Also used : ObjectNode(io.openk9.json.api.ObjectNode) JsonNode(io.openk9.json.api.JsonNode)

Example 4 with DatasourceContext

use of io.openk9.model.DatasourceContext in project openk9 by smclab.

the class DatasourceProcessor method process.

@Incoming("ingestion")
@Outgoing("ingestion-datasource")
public Uni<IngestionDatasourcePayload> process(Object obj) {
    JsonObject jsonObject = obj instanceof JsonObject ? (JsonObject) obj : new JsonObject(new String((byte[]) obj));
    long datasourceId = jsonObject.getLong("datasourceId");
    Uni<Datasource> datasourceUni = Datasource.findById(datasourceId);
    return Panache.withTransaction(() -> datasourceUni.flatMap(datasource -> EnrichPipeline.findByDatasourceId(datasource.getDatasourceId()).onItem().ifNull().continueWith(EnrichPipeline::new).flatMap(enrichPipeline -> {
        Uni<List<EnrichItem>> enrichItemUni;
        if (enrichPipeline.getEnrichPipelineId() != null) {
            enrichItemUni = EnrichItem.findByEnrichPipelineId(enrichPipeline.getEnrichPipelineId()).onItem().ifNull().continueWith(List::of);
        } else {
            enrichItemUni = Uni.createFrom().item(List.of());
        }
        return Uni.combine().all().unis(Tenant.findById(datasource.getTenantId()), enrichItemUni).combinedWith((tenantObj, enrichItemList) -> {
            Tenant tenant = (Tenant) tenantObj;
            IngestionPayload ingestionPayload = jsonObject.mapTo(IngestionPayload.class);
            ingestionPayload.setTenantId(tenant.getTenantId());
            DatasourceContext datasourceContext = DatasourceContext.of(datasource, tenant, enrichPipeline, enrichItemList);
            return IngestionDatasourcePayload.of(ingestionPayload, datasourceContext);
        });
    })).eventually(() -> Datasource.<Datasource>findById(datasourceId).flatMap(datasource -> {
        datasource.setLastIngestionDate(Instant.ofEpochMilli(jsonObject.getLong("parsingDate")));
        return datasource.persist();
    })));
}
Also used : Datasource(io.openk9.datasource.model.Datasource) Outgoing(org.eclipse.microprofile.reactive.messaging.Outgoing) Datasource(io.openk9.datasource.model.Datasource) EnrichItem(io.openk9.datasource.model.EnrichItem) Tenant(io.openk9.datasource.model.Tenant) Instant(java.time.Instant) Uni(io.smallrye.mutiny.Uni) DatasourceContext(io.openk9.datasource.processor.payload.DatasourceContext) List(java.util.List) IngestionDatasourcePayload(io.openk9.datasource.processor.payload.IngestionDatasourcePayload) IngestionPayload(io.openk9.datasource.processor.payload.IngestionPayload) EnrichPipeline(io.openk9.datasource.model.EnrichPipeline) JsonObject(io.vertx.core.json.JsonObject) ApplicationScoped(javax.enterprise.context.ApplicationScoped) Panache(io.quarkus.hibernate.reactive.panache.Panache) Incoming(org.eclipse.microprofile.reactive.messaging.Incoming) JsonObject(io.vertx.core.json.JsonObject) EnrichItem(io.openk9.datasource.model.EnrichItem) Uni(io.smallrye.mutiny.Uni) Tenant(io.openk9.datasource.model.Tenant) IngestionPayload(io.openk9.datasource.processor.payload.IngestionPayload) DatasourceContext(io.openk9.datasource.processor.payload.DatasourceContext) EnrichPipeline(io.openk9.datasource.model.EnrichPipeline) List(java.util.List) Incoming(org.eclipse.microprofile.reactive.messaging.Incoming) Outgoing(org.eclipse.microprofile.reactive.messaging.Outgoing)

Example 5 with DatasourceContext

use of io.openk9.model.DatasourceContext in project openk9 by smclab.

the class JsEnrichProcessor method prepareRequestRawContent.

protected ObjectNode prepareRequestRawContent(ObjectNode objectNode, ObjectNode datasourceConfiguration, DatasourceContext context, PluginDriverDTO pluginDriverDTO) {
    JsonNode rawContentNode = objectNode.get(Constants.RAW_CONTENT);
    JsonNode codeNode = datasourceConfiguration.get(Constants.CODE);
    ObjectNode request = _jsonFactory.createObjectNode();
    request.put(Constants.CODE, codeNode);
    request.put(Constants.CONTENT, rawContentNode);
    JsonNode typeNode = objectNode.get(Constants.TYPE);
    ObjectNode datasourcePayload = _jsonFactory.createObjectNode();
    if (typeNode != null && typeNode.isArray()) {
        ArrayNode types = typeNode.toArrayNode();
        for (JsonNode typeJsonNode : types) {
            String type = typeJsonNode.asText();
            datasourcePayload.put(type, objectNode.get(type));
        }
    }
    request.put(Constants.DATASOURCE_PAYLOAD, datasourcePayload);
    request.put(Constants.TENANT_ID, context.getTenant().getTenantId());
    request.put(Constants.DATASOURCE_ID, context.getDatasource().getDatasourceId());
    request.put(Constants.CONTENT_ID, objectNode.get(Constants.CONTENT_ID));
    return request;
}
Also used : ObjectNode(io.openk9.json.api.ObjectNode) JsonNode(io.openk9.json.api.JsonNode) ArrayNode(io.openk9.json.api.ArrayNode)

Aggregations

ObjectNode (io.openk9.json.api.ObjectNode)8 JsonNode (io.openk9.json.api.JsonNode)6 ArrayNode (io.openk9.json.api.ArrayNode)3 List (java.util.List)2 Activate (org.osgi.service.component.annotations.Activate)2 Datasource (io.openk9.datasource.model.Datasource)1 EnrichItem (io.openk9.datasource.model.EnrichItem)1 EnrichPipeline (io.openk9.datasource.model.EnrichPipeline)1 Tenant (io.openk9.datasource.model.Tenant)1 DatasourceContext (io.openk9.datasource.processor.payload.DatasourceContext)1 IngestionDatasourcePayload (io.openk9.datasource.processor.payload.IngestionDatasourcePayload)1 IngestionPayload (io.openk9.datasource.processor.payload.IngestionPayload)1 Entity (io.openk9.entity.manager.model.Entity)1 EntityRequest (io.openk9.entity.manager.model.payload.EntityRequest)1 RelationRequest (io.openk9.entity.manager.model.payload.RelationRequest)1 Request (io.openk9.entity.manager.model.payload.Request)1 Response (io.openk9.entity.manager.model.payload.Response)1 ResponseList (io.openk9.entity.manager.model.payload.ResponseList)1 Binding (io.openk9.ingestion.api.Binding)1 OutboundMessageFactory (io.openk9.ingestion.api.OutboundMessageFactory)1