use of io.openk9.json.api.JsonNode in project openk9 by smclab.
the class GetOrAddEntitiesConsumer method apply.
public Mono<ObjectNode> apply(ObjectNode objectNode) {
return Mono.defer(() -> {
ObjectNode datasourceContextJson = objectNode.get("datasourceContext").toObjectNode();
long datasourceId = datasourceContextJson.get("datasource").get("datasourceId").asLong();
long tenantId = datasourceContextJson.get("tenant").get("tenantId").asLong();
JsonNode entities = objectNode.remove("entities");
Mono<ArrayNode> entitiesField;
if (entities.size() == 0) {
entitiesField = Mono.just(_jsonFactory.createArrayNode());
} else {
ObjectNode responseJson = _jsonFactory.createObjectNode();
responseJson.put("entities", entities);
responseJson.put("tenantId", tenantId);
responseJson.put("datasourceId", datasourceId);
Request request = _jsonFactory.fromJson(responseJson.toString(), Request.class);
List<RequestContext> requestContextList = request.getEntities().stream().map(entityRequest -> RequestContext.builder().current(entityRequest).tenantId(request.getTenantId()).datasourceId(request.getDatasourceId()).rest(request.getEntities().stream().filter(er -> er != entityRequest).collect(Collectors.toList())).build()).collect(Collectors.toList());
Mono<List<EntityContext>> disambiguateListMono = GetOrAddEntities.stopWatch("disambiguate-all-entities", Flux.fromIterable(requestContextList).flatMap(requestContext -> GetOrAddEntities.stopWatch("disambiguate-" + requestContext.getCurrent().getName(), Mono.<EntityContext>create(fluxSink -> _startDisambiguation.disambiguate(requestContext, fluxSink)))).collectList());
Mono<ResponseList> writeRelations = disambiguateListMono.flatMap(entityContexts -> GetOrAddEntities.stopWatch("write-relations", writeRelations(entityContexts)));
Mono<ResponseList> responseListWrapper = _transactional ? _graphClient.makeTransactional(writeRelations) : writeRelations;
entitiesField = responseListWrapper.map(responseListDTO -> {
List<Response> responseList = responseListDTO.getResponse();
ArrayNode entitiesArrayNode = entities.toArrayNode();
ArrayNode arrayNode = _jsonFactory.createArrayNode();
for (JsonNode node : entitiesArrayNode) {
Optional<Response> responseOptional = responseList.stream().filter(response -> node.get("tmpId").asLong() == response.getTmpId()).findFirst();
if (responseOptional.isPresent()) {
Entity entity = responseOptional.get().getEntity();
ObjectNode result = _jsonFactory.createObjectNode();
result.put("entityType", entity.getType());
result.put("id", entity.getId());
result.put("context", node.get("context"));
arrayNode.add(result);
}
}
return arrayNode;
});
}
return entitiesField.map(entitiesArray -> {
ObjectNode payload = objectNode.get("payload").toObjectNode();
payload.set("entities", entitiesArray);
return objectNode;
}).timeout(Duration.ofSeconds(_timeout), Mono.error(new TimeoutException("timeout on entities count: " + entities.size() + " (Did not observe any item or terminal signal within " + Duration.ofSeconds(_timeout).toMillis() + "ms)")));
});
}
use of io.openk9.json.api.JsonNode in project openk9 by smclab.
the class EntitySearchHTTPHandler method _toSearchRequest.
private SearchRequest _toSearchRequest(Long tenantId, ObjectNode jsonNodes) {
JsonNode sizeJsonNode = jsonNodes.remove("size");
BoolQueryBuilder boolQuery = _getBoolQueryBuilder(jsonNodes);
SearchRequest searchRequest = _searchRequestFactory.createSearchRequestEntity(tenantId);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
if (sizeJsonNode != null && sizeJsonNode.isArray()) {
ArrayNode sizeArrayNode = sizeJsonNode.toArrayNode();
int length = sizeArrayNode.size();
if (length == 2) {
JsonNode fromJsonNode = sizeArrayNode.get(0);
JsonNode sizeJN = sizeArrayNode.get(1);
int size = sizeJN.asInt();
if (size > 10000) {
if (_log.isWarnEnabled()) {
_log.warn("size[1] max size is 10000");
}
size = 10000;
}
searchSourceBuilder.from(fromJsonNode.asInt());
searchSourceBuilder.size(size);
}
}
searchSourceBuilder.query(boolQuery);
return searchRequest.source(searchSourceBuilder);
}
use of io.openk9.json.api.JsonNode in project openk9 by smclab.
the class BaseNerEnrichProcessor method prepareRequestRawContent.
protected ObjectNode prepareRequestRawContent(ObjectNode objectNode, ObjectNode datasourceConfiguration, DatasourceContext context, PluginDriverDTO pluginDriverDTO) {
JsonNode entitiesNode = datasourceConfiguration.get(entitiesField());
JsonNode relationsNode = datasourceConfiguration.get(relationsField());
JsonNode rawContentNode = objectNode.get(Constants.RAW_CONTENT);
JsonNode confidenceNode = datasourceConfiguration.get(Constants.CONFIDENCE);
ObjectNode request = _jsonFactory.createObjectNode();
request.put(entitiesField(), entitiesNode);
request.put(relationsField(), relationsNode);
request.put(Constants.CONFIDENCE, confidenceNode);
request.put(Constants.CONTENT, rawContentNode);
JsonNode typeNode = objectNode.get(Constants.TYPE);
ObjectNode datasourcePayload = _jsonFactory.createObjectNode();
if (typeNode != null && typeNode.isArray()) {
ArrayNode types = typeNode.toArrayNode();
for (JsonNode typeJsonNode : types) {
String type = typeJsonNode.asText();
datasourcePayload.put(type, objectNode.get(type));
}
}
request.put(Constants.DATASOURCE_PAYLOAD, datasourcePayload);
request.put(Constants.TENANT_ID, context.getTenant().getTenantId());
request.put(Constants.DATASOURCE_ID, context.getDatasource().getDatasourceId());
request.put(Constants.CONTENT_ID, objectNode.get(Constants.CONTENT_ID));
return request;
}
use of io.openk9.json.api.JsonNode in project openk9 by smclab.
the class GrammarProvider method _toJavaRules.
private List<Rule> _toJavaRules(JsonNode jsonNode) {
ArrayNode arrayNode = jsonNode.toArrayNode();
List<Rule> rules = new ArrayList<>();
for (JsonNode node : arrayNode) {
String lhs = node.get("lhs").asText();
String rhs = node.get("rhs").asText();
rules.add(Rule.of(lhs, rhs, Semantic.identity()));
}
return rules;
}
use of io.openk9.json.api.JsonNode in project openk9 by smclab.
the class ApplicationPluginDriver method invokeDataParser.
@Override
public Publisher<Void> invokeDataParser(Datasource datasource, Date fromDate, Date toDate) {
JsonNode jsonNode = _jsonFactory.fromJsonToJsonNode(datasource.getJsonConfig());
if (jsonNode.isArray()) {
ArrayNode arrayJson = jsonNode.toArrayNode();
for (int i = 0; i < arrayJson.size(); i++) {
JsonNode node = arrayJson.get(i);
if (node.hasNonNull("title")) {
node = node.toObjectNode().set("applicationName", node.get("title"));
}
_ingestionLogicSender.send(IngestionPayload.builder().ingestionId(UUID.randomUUID().toString()).datasourceId(datasource.getDatasourceId()).rawContent(node.toString()).contentId(Integer.toString(i)).tenantId(datasource.getTenantId()).datasourcePayload(_jsonFactory.createObjectNode().set(getName(), node.toObjectNode()).toMap()).parsingDate(toDate.getTime()).documentTypes(new String[] { getName() }).build());
}
}
return Mono.empty();
}
Aggregations