Search in sources :

Example 1 with JsonNode

use of io.openk9.json.api.JsonNode in project openk9 by smclab.

the class GetOrAddEntitiesConsumer method apply.

public Mono<ObjectNode> apply(ObjectNode objectNode) {
    return Mono.defer(() -> {
        ObjectNode datasourceContextJson = objectNode.get("datasourceContext").toObjectNode();
        long datasourceId = datasourceContextJson.get("datasource").get("datasourceId").asLong();
        long tenantId = datasourceContextJson.get("tenant").get("tenantId").asLong();
        JsonNode entities = objectNode.remove("entities");
        Mono<ArrayNode> entitiesField;
        if (entities.size() == 0) {
            entitiesField = Mono.just(_jsonFactory.createArrayNode());
        } else {
            ObjectNode responseJson = _jsonFactory.createObjectNode();
            responseJson.put("entities", entities);
            responseJson.put("tenantId", tenantId);
            responseJson.put("datasourceId", datasourceId);
            Request request = _jsonFactory.fromJson(responseJson.toString(), Request.class);
            List<RequestContext> requestContextList = request.getEntities().stream().map(entityRequest -> RequestContext.builder().current(entityRequest).tenantId(request.getTenantId()).datasourceId(request.getDatasourceId()).rest(request.getEntities().stream().filter(er -> er != entityRequest).collect(Collectors.toList())).build()).collect(Collectors.toList());
            Mono<List<EntityContext>> disambiguateListMono = GetOrAddEntities.stopWatch("disambiguate-all-entities", Flux.fromIterable(requestContextList).flatMap(requestContext -> GetOrAddEntities.stopWatch("disambiguate-" + requestContext.getCurrent().getName(), Mono.<EntityContext>create(fluxSink -> _startDisambiguation.disambiguate(requestContext, fluxSink)))).collectList());
            Mono<ResponseList> writeRelations = disambiguateListMono.flatMap(entityContexts -> GetOrAddEntities.stopWatch("write-relations", writeRelations(entityContexts)));
            Mono<ResponseList> responseListWrapper = _transactional ? _graphClient.makeTransactional(writeRelations) : writeRelations;
            entitiesField = responseListWrapper.map(responseListDTO -> {
                List<Response> responseList = responseListDTO.getResponse();
                ArrayNode entitiesArrayNode = entities.toArrayNode();
                ArrayNode arrayNode = _jsonFactory.createArrayNode();
                for (JsonNode node : entitiesArrayNode) {
                    Optional<Response> responseOptional = responseList.stream().filter(response -> node.get("tmpId").asLong() == response.getTmpId()).findFirst();
                    if (responseOptional.isPresent()) {
                        Entity entity = responseOptional.get().getEntity();
                        ObjectNode result = _jsonFactory.createObjectNode();
                        result.put("entityType", entity.getType());
                        result.put("id", entity.getId());
                        result.put("context", node.get("context"));
                        arrayNode.add(result);
                    }
                }
                return arrayNode;
            });
        }
        return entitiesField.map(entitiesArray -> {
            ObjectNode payload = objectNode.get("payload").toObjectNode();
            payload.set("entities", entitiesArray);
            return objectNode;
        }).timeout(Duration.ofSeconds(_timeout), Mono.error(new TimeoutException("timeout on entities count: " + entities.size() + " (Did not observe any item or terminal signal within " + Duration.ofSeconds(_timeout).toMillis() + "ms)")));
    });
}
Also used : Response(io.openk9.entity.manager.model.payload.Response) SenderReactor(io.openk9.ingestion.api.SenderReactor) Disposable(reactor.core.Disposable) JsonNode(io.openk9.json.api.JsonNode) LoggerFactory(org.slf4j.LoggerFactory) Tuples(reactor.util.function.Tuples) Tuple2(reactor.util.function.Tuple2) TimeoutException(java.util.concurrent.TimeoutException) EntityRequest(io.openk9.entity.manager.model.payload.EntityRequest) GraphClient(io.openk9.relationship.graph.api.client.GraphClient) ArrayNode(io.openk9.json.api.ArrayNode) JsonFactory(io.openk9.json.api.JsonFactory) ArrayList(java.util.ArrayList) Component(org.osgi.service.component.annotations.Component) Statement(org.neo4j.cypherdsl.core.Statement) Duration(java.time.Duration) OutboundMessageFactory(io.openk9.ingestion.api.OutboundMessageFactory) Schedulers(reactor.core.scheduler.Schedulers) Request(io.openk9.entity.manager.model.payload.Request) Cypher(org.neo4j.cypherdsl.core.Cypher) Activate(org.osgi.service.component.annotations.Activate) Node(org.neo4j.cypherdsl.core.Node) Binding(io.openk9.ingestion.api.Binding) Logger(org.slf4j.Logger) Deactivate(org.osgi.service.component.annotations.Deactivate) Functions(org.neo4j.cypherdsl.core.Functions) RelationRequest(io.openk9.entity.manager.model.payload.RelationRequest) Mono(reactor.core.publisher.Mono) Entity(io.openk9.entity.manager.model.Entity) Collectors(java.util.stream.Collectors) Flux(reactor.core.publisher.Flux) List(java.util.List) Stream(java.util.stream.Stream) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ReceiverReactor(io.openk9.ingestion.api.ReceiverReactor) Modified(org.osgi.service.component.annotations.Modified) Optional(java.util.Optional) ObjectNode(io.openk9.json.api.ObjectNode) Reference(org.osgi.service.component.annotations.Reference) Cypher.literalOf(org.neo4j.cypherdsl.core.Cypher.literalOf) Entity(io.openk9.entity.manager.model.Entity) ObjectNode(io.openk9.json.api.ObjectNode) Optional(java.util.Optional) EntityRequest(io.openk9.entity.manager.model.payload.EntityRequest) Request(io.openk9.entity.manager.model.payload.Request) RelationRequest(io.openk9.entity.manager.model.payload.RelationRequest) JsonNode(io.openk9.json.api.JsonNode) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ArrayList(java.util.ArrayList) List(java.util.List) ResponseList(io.openk9.entity.manager.model.payload.ResponseList) ArrayNode(io.openk9.json.api.ArrayNode) TimeoutException(java.util.concurrent.TimeoutException)

Example 2 with JsonNode

use of io.openk9.json.api.JsonNode in project openk9 by smclab.

the class EntitySearchHTTPHandler method _toSearchRequest.

private SearchRequest _toSearchRequest(Long tenantId, ObjectNode jsonNodes) {
    JsonNode sizeJsonNode = jsonNodes.remove("size");
    BoolQueryBuilder boolQuery = _getBoolQueryBuilder(jsonNodes);
    SearchRequest searchRequest = _searchRequestFactory.createSearchRequestEntity(tenantId);
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    if (sizeJsonNode != null && sizeJsonNode.isArray()) {
        ArrayNode sizeArrayNode = sizeJsonNode.toArrayNode();
        int length = sizeArrayNode.size();
        if (length == 2) {
            JsonNode fromJsonNode = sizeArrayNode.get(0);
            JsonNode sizeJN = sizeArrayNode.get(1);
            int size = sizeJN.asInt();
            if (size > 10000) {
                if (_log.isWarnEnabled()) {
                    _log.warn("size[1] max size is 10000");
                }
                size = 10000;
            }
            searchSourceBuilder.from(fromJsonNode.asInt());
            searchSourceBuilder.size(size);
        }
    }
    searchSourceBuilder.query(boolQuery);
    return searchRequest.source(searchSourceBuilder);
}
Also used : SearchRequest(org.elasticsearch.action.search.SearchRequest) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) JsonNode(io.openk9.json.api.JsonNode) ArrayNode(io.openk9.json.api.ArrayNode) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder)

Example 3 with JsonNode

use of io.openk9.json.api.JsonNode in project openk9 by smclab.

the class BaseNerEnrichProcessor method prepareRequestRawContent.

protected ObjectNode prepareRequestRawContent(ObjectNode objectNode, ObjectNode datasourceConfiguration, DatasourceContext context, PluginDriverDTO pluginDriverDTO) {
    JsonNode entitiesNode = datasourceConfiguration.get(entitiesField());
    JsonNode relationsNode = datasourceConfiguration.get(relationsField());
    JsonNode rawContentNode = objectNode.get(Constants.RAW_CONTENT);
    JsonNode confidenceNode = datasourceConfiguration.get(Constants.CONFIDENCE);
    ObjectNode request = _jsonFactory.createObjectNode();
    request.put(entitiesField(), entitiesNode);
    request.put(relationsField(), relationsNode);
    request.put(Constants.CONFIDENCE, confidenceNode);
    request.put(Constants.CONTENT, rawContentNode);
    JsonNode typeNode = objectNode.get(Constants.TYPE);
    ObjectNode datasourcePayload = _jsonFactory.createObjectNode();
    if (typeNode != null && typeNode.isArray()) {
        ArrayNode types = typeNode.toArrayNode();
        for (JsonNode typeJsonNode : types) {
            String type = typeJsonNode.asText();
            datasourcePayload.put(type, objectNode.get(type));
        }
    }
    request.put(Constants.DATASOURCE_PAYLOAD, datasourcePayload);
    request.put(Constants.TENANT_ID, context.getTenant().getTenantId());
    request.put(Constants.DATASOURCE_ID, context.getDatasource().getDatasourceId());
    request.put(Constants.CONTENT_ID, objectNode.get(Constants.CONTENT_ID));
    return request;
}
Also used : ObjectNode(io.openk9.json.api.ObjectNode) JsonNode(io.openk9.json.api.JsonNode) ArrayNode(io.openk9.json.api.ArrayNode)

Example 4 with JsonNode

use of io.openk9.json.api.JsonNode in project openk9 by smclab.

the class GrammarProvider method _toJavaRules.

private List<Rule> _toJavaRules(JsonNode jsonNode) {
    ArrayNode arrayNode = jsonNode.toArrayNode();
    List<Rule> rules = new ArrayList<>();
    for (JsonNode node : arrayNode) {
        String lhs = node.get("lhs").asText();
        String rhs = node.get("rhs").asText();
        rules.add(Rule.of(lhs, rhs, Semantic.identity()));
    }
    return rules;
}
Also used : ArrayList(java.util.ArrayList) JsonNode(io.openk9.json.api.JsonNode) ArrayNode(io.openk9.json.api.ArrayNode)

Example 5 with JsonNode

use of io.openk9.json.api.JsonNode in project openk9 by smclab.

the class ApplicationPluginDriver method invokeDataParser.

@Override
public Publisher<Void> invokeDataParser(Datasource datasource, Date fromDate, Date toDate) {
    JsonNode jsonNode = _jsonFactory.fromJsonToJsonNode(datasource.getJsonConfig());
    if (jsonNode.isArray()) {
        ArrayNode arrayJson = jsonNode.toArrayNode();
        for (int i = 0; i < arrayJson.size(); i++) {
            JsonNode node = arrayJson.get(i);
            if (node.hasNonNull("title")) {
                node = node.toObjectNode().set("applicationName", node.get("title"));
            }
            _ingestionLogicSender.send(IngestionPayload.builder().ingestionId(UUID.randomUUID().toString()).datasourceId(datasource.getDatasourceId()).rawContent(node.toString()).contentId(Integer.toString(i)).tenantId(datasource.getTenantId()).datasourcePayload(_jsonFactory.createObjectNode().set(getName(), node.toObjectNode()).toMap()).parsingDate(toDate.getTime()).documentTypes(new String[] { getName() }).build());
        }
    }
    return Mono.empty();
}
Also used : JsonNode(io.openk9.json.api.JsonNode) ArrayNode(io.openk9.json.api.ArrayNode)

Aggregations

JsonNode (io.openk9.json.api.JsonNode)12 ObjectNode (io.openk9.json.api.ObjectNode)8 ArrayNode (io.openk9.json.api.ArrayNode)7 JsonFactory (io.openk9.json.api.JsonFactory)3 Map (java.util.Map)3 Mono (reactor.core.publisher.Mono)3 Entity (io.openk9.entity.manager.model.Entity)2 Request (io.openk9.entity.manager.model.payload.Request)2 Response (io.openk9.entity.manager.model.payload.Response)2 HttpClient (io.openk9.http.client.HttpClient)2 ArrayList (java.util.ArrayList)2 List (java.util.List)2 Optional (java.util.Optional)2 Collectors (java.util.stream.Collectors)2 Activate (org.osgi.service.component.annotations.Activate)2 Constants (io.openk9.core.api.constant.Constants)1 EntityManagerClient (io.openk9.entity.manager.client.api.EntityManagerClient)1 EntityRequest (io.openk9.entity.manager.model.payload.EntityRequest)1 RelationRequest (io.openk9.entity.manager.model.payload.RelationRequest)1 ResponseList (io.openk9.entity.manager.model.payload.ResponseList)1