use of io.openk9.entity.manager.model.payload.Request in project openk9 by smclab.
the class GetOrAddEntitiesConsumer method apply.
public Mono<ObjectNode> apply(ObjectNode objectNode) {
return Mono.defer(() -> {
ObjectNode datasourceContextJson = objectNode.get("datasourceContext").toObjectNode();
long datasourceId = datasourceContextJson.get("datasource").get("datasourceId").asLong();
long tenantId = datasourceContextJson.get("tenant").get("tenantId").asLong();
JsonNode entities = objectNode.remove("entities");
Mono<ArrayNode> entitiesField;
if (entities.size() == 0) {
entitiesField = Mono.just(_jsonFactory.createArrayNode());
} else {
ObjectNode responseJson = _jsonFactory.createObjectNode();
responseJson.put("entities", entities);
responseJson.put("tenantId", tenantId);
responseJson.put("datasourceId", datasourceId);
Request request = _jsonFactory.fromJson(responseJson.toString(), Request.class);
List<RequestContext> requestContextList = request.getEntities().stream().map(entityRequest -> RequestContext.builder().current(entityRequest).tenantId(request.getTenantId()).datasourceId(request.getDatasourceId()).rest(request.getEntities().stream().filter(er -> er != entityRequest).collect(Collectors.toList())).build()).collect(Collectors.toList());
Mono<List<EntityContext>> disambiguateListMono = GetOrAddEntities.stopWatch("disambiguate-all-entities", Flux.fromIterable(requestContextList).flatMap(requestContext -> GetOrAddEntities.stopWatch("disambiguate-" + requestContext.getCurrent().getName(), Mono.<EntityContext>create(fluxSink -> _startDisambiguation.disambiguate(requestContext, fluxSink)))).collectList());
Mono<ResponseList> writeRelations = disambiguateListMono.flatMap(entityContexts -> GetOrAddEntities.stopWatch("write-relations", writeRelations(entityContexts)));
Mono<ResponseList> responseListWrapper = _transactional ? _graphClient.makeTransactional(writeRelations) : writeRelations;
entitiesField = responseListWrapper.map(responseListDTO -> {
List<Response> responseList = responseListDTO.getResponse();
ArrayNode entitiesArrayNode = entities.toArrayNode();
ArrayNode arrayNode = _jsonFactory.createArrayNode();
for (JsonNode node : entitiesArrayNode) {
Optional<Response> responseOptional = responseList.stream().filter(response -> node.get("tmpId").asLong() == response.getTmpId()).findFirst();
if (responseOptional.isPresent()) {
Entity entity = responseOptional.get().getEntity();
ObjectNode result = _jsonFactory.createObjectNode();
result.put("entityType", entity.getType());
result.put("id", entity.getId());
result.put("context", node.get("context"));
arrayNode.add(result);
}
}
return arrayNode;
});
}
return entitiesField.map(entitiesArray -> {
ObjectNode payload = objectNode.get("payload").toObjectNode();
payload.set("entities", entitiesArray);
return objectNode;
}).timeout(Duration.ofSeconds(_timeout), Mono.error(new TimeoutException("timeout on entities count: " + entities.size() + " (Did not observe any item or terminal signal within " + Duration.ofSeconds(_timeout).toMillis() + "ms)")));
});
}
use of io.openk9.entity.manager.model.payload.Request in project openk9 by smclab.
the class IndexWriterEndpoins method _insertEntity.
private Publisher<Void> _insertEntity(HttpServerRequest httpRequest, HttpServerResponse httpResponse) {
RestHighLevelClient restHighLevelClient = _restHighLevelClientProvider.get();
Mono<List<DocumentEntityRequest>> request = Mono.from(ReactorNettyUtils.aggregateBodyAsByteArray(httpRequest)).map(json -> _jsonFactory.fromJsonList(json, DocumentEntityRequest.class));
Mono<BulkResponse> elasticResponse = request.flatMapIterable(Function.identity()).map(entity -> {
IndexRequest indexRequest = new IndexRequest(entity.getTenantId() + "-entity");
return indexRequest.source(_jsonFactory.toJson(entity), XContentType.JSON);
}).reduce(new BulkRequest(), BulkRequest::add).flatMap(bulkRequest -> Mono.create(sink -> {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
Cancellable cancellable = restHighLevelClient.bulkAsync(bulkRequest, RequestOptions.DEFAULT, new ReactorActionListener<>(sink));
sink.onCancel(cancellable::cancel);
}));
return _httpResponseWriter.write(httpResponse, elasticResponse.thenReturn("{}"));
}
use of io.openk9.entity.manager.model.payload.Request in project openk9 by smclab.
the class EntityManagerBus method run.
@SneakyThrows
public void run() {
while (true) {
Payload request = _entityManagerQueue.take();
TransactionContext transactionContext = _hazelcastInstance.newTransactionContext();
transactionContext.beginTransaction();
try {
TransactionalMap<EntityKey, Entity> entityTransactionalMap = transactionContext.getMap("entityMap");
TransactionalMap<EntityRelationKey, EntityRelation> transactionalEntityRelationMap = transactionContext.getMap("entityRelationMap");
TransactionalMultiMap<DocumentKey, String> documentEntityMap = transactionContext.getMultiMap("documentEntityMap");
EntityManagerRequest payload = request.getPayload();
_loggerAggregator.emitLog("process ingestionId", payload.getIngestionId());
long tenantId = payload.getTenantId();
String ingestionId = payload.getIngestionId();
List<EntityRequest> entities = request.getEntities();
Map<EntityKey, Entity> localEntityMap = new HashMap<>(entities.size());
for (EntityRequest entityRequest : entities) {
String name = entityRequest.getName();
String type = entityRequest.getType();
String cacheId = Long.toString(_entityFlakeId.newId());
EntityKey key = EntityKey.of(tenantId, name, type, cacheId, ingestionId);
Entity entity = new Entity(null, cacheId, tenantId, name, type, null, ingestionId, false, true, entityRequest.getContext());
entityTransactionalMap.set(key, entity);
localEntityMap.put(key, entity);
for (EntityRequest entityRequest2 : entities) {
for (RelationRequest relation : entityRequest2.getRelations()) {
if (relation.getTo().equals(entityRequest.getTmpId())) {
relation.setTo(entity.getCacheId());
}
}
}
}
for (EntityRequest entity : entities) {
List<RelationRequest> relations = entity.getRelations();
if (relations == null || relations.isEmpty()) {
continue;
}
Collection<Entity> values = localEntityMap.values();
Entity current = values.stream().filter(e -> e.getName().equals(entity.getName()) && e.getType().equals(entity.getType())).findFirst().orElse(null);
if (current == null) {
continue;
}
for (RelationRequest relation : relations) {
String to = relation.getTo();
String name = relation.getName();
for (Entity value : values) {
if (value.getCacheId().equals(to)) {
long entityRelationId = _entityRelationFlakeId.newId();
EntityRelation entityRelation = new EntityRelation(entityRelationId, current.getCacheId(), ingestionId, name, value.getCacheId());
transactionalEntityRelationMap.set(EntityRelationKey.of(entityRelationId, current.getCacheId(), ingestionId), entityRelation);
}
}
}
}
if (!localEntityMap.isEmpty()) {
DocumentKey key = DocumentKey.of(payload.getDatasourceId(), payload.getContentId(), tenantId);
for (Entity value : localEntityMap.values()) {
documentEntityMap.put(key, value.getCacheId());
}
}
} catch (Exception e) {
_log.error(e.getMessage(), e);
transactionContext.rollbackTransaction();
} finally {
transactionContext.commitTransaction();
}
}
}
use of io.openk9.entity.manager.model.payload.Request in project openk9 by smclab.
the class BaseNerEnrichProcessor method prepareRequestRawContent.
protected ObjectNode prepareRequestRawContent(ObjectNode objectNode, ObjectNode datasourceConfiguration, DatasourceContext context, PluginDriverDTO pluginDriverDTO) {
JsonNode entitiesNode = datasourceConfiguration.get(entitiesField());
JsonNode relationsNode = datasourceConfiguration.get(relationsField());
JsonNode rawContentNode = objectNode.get(Constants.RAW_CONTENT);
JsonNode confidenceNode = datasourceConfiguration.get(Constants.CONFIDENCE);
ObjectNode request = _jsonFactory.createObjectNode();
request.put(entitiesField(), entitiesNode);
request.put(relationsField(), relationsNode);
request.put(Constants.CONFIDENCE, confidenceNode);
request.put(Constants.CONTENT, rawContentNode);
JsonNode typeNode = objectNode.get(Constants.TYPE);
ObjectNode datasourcePayload = _jsonFactory.createObjectNode();
if (typeNode != null && typeNode.isArray()) {
ArrayNode types = typeNode.toArrayNode();
for (JsonNode typeJsonNode : types) {
String type = typeJsonNode.asText();
datasourcePayload.put(type, objectNode.get(type));
}
}
request.put(Constants.DATASOURCE_PAYLOAD, datasourcePayload);
request.put(Constants.TENANT_ID, context.getTenant().getTenantId());
request.put(Constants.DATASOURCE_ID, context.getDatasource().getDatasourceId());
request.put(Constants.CONTENT_ID, objectNode.get(Constants.CONTENT_ID));
return request;
}
use of io.openk9.entity.manager.model.payload.Request in project openk9 by smclab.
the class JsEnrichProcessor method process.
@Override
public Mono<ObjectNode> process(ObjectNode objectNode, DatasourceContext context, EnrichItem enrichItem, PluginDriverDTO pluginDriverName) {
return Mono.defer(() -> {
JsonNode datasourceConfiguration = _jsonFactory.fromJsonToJsonNode(enrichItem.getJsonConfig());
if (!datasourceConfiguration.isObject()) {
return Mono.error(new RuntimeException("jsonConfig must be an instance of ObjectNode " + datasourceConfiguration.toString()));
}
ObjectNode request = prepareRequestRawContent(objectNode, datasourceConfiguration.toObjectNode(), context, pluginDriverName);
return Mono.from(_httpClient.request(getMethod(), getPath(), request.toString(), getHeaders())).map(_jsonFactory::fromJsonToJsonNode).map(JsonNode::toObjectNode).map(objectNode::merge);
});
}
Aggregations