Examples with Payload io.openk9.entity.manager.dto.Payload used on opensource projects

Example 1 with Payload

use of io.openk9.entity.manager.dto.Payload in project openk9 by smclab.

the class GetOrAddEntitiesConsumer method apply.

public Mono<ObjectNode> apply(ObjectNode objectNode) {
    return Mono.defer(() -> {
        ObjectNode datasourceContextJson = objectNode.get("datasourceContext").toObjectNode();
        long datasourceId = datasourceContextJson.get("datasource").get("datasourceId").asLong();
        long tenantId = datasourceContextJson.get("tenant").get("tenantId").asLong();
        JsonNode entities = objectNode.remove("entities");
        Mono<ArrayNode> entitiesField;
        if (entities.size() == 0) {
            entitiesField = Mono.just(_jsonFactory.createArrayNode());
        } else {
            ObjectNode responseJson = _jsonFactory.createObjectNode();
            responseJson.put("entities", entities);
            responseJson.put("tenantId", tenantId);
            responseJson.put("datasourceId", datasourceId);
            Request request = _jsonFactory.fromJson(responseJson.toString(), Request.class);
            List<RequestContext> requestContextList = request.getEntities().stream().map(entityRequest -> RequestContext.builder().current(entityRequest).tenantId(request.getTenantId()).datasourceId(request.getDatasourceId()).rest(request.getEntities().stream().filter(er -> er != entityRequest).collect(Collectors.toList())).build()).collect(Collectors.toList());
            Mono<List<EntityContext>> disambiguateListMono = GetOrAddEntities.stopWatch("disambiguate-all-entities", Flux.fromIterable(requestContextList).flatMap(requestContext -> GetOrAddEntities.stopWatch("disambiguate-" + requestContext.getCurrent().getName(), Mono.<EntityContext>create(fluxSink -> _startDisambiguation.disambiguate(requestContext, fluxSink)))).collectList());
            Mono<ResponseList> writeRelations = disambiguateListMono.flatMap(entityContexts -> GetOrAddEntities.stopWatch("write-relations", writeRelations(entityContexts)));
            Mono<ResponseList> responseListWrapper = _transactional ? _graphClient.makeTransactional(writeRelations) : writeRelations;
            entitiesField = responseListWrapper.map(responseListDTO -> {
                List<Response> responseList = responseListDTO.getResponse();
                ArrayNode entitiesArrayNode = entities.toArrayNode();
                ArrayNode arrayNode = _jsonFactory.createArrayNode();
                for (JsonNode node : entitiesArrayNode) {
                    Optional<Response> responseOptional = responseList.stream().filter(response -> node.get("tmpId").asLong() == response.getTmpId()).findFirst();
                    if (responseOptional.isPresent()) {
                        Entity entity = responseOptional.get().getEntity();
                        ObjectNode result = _jsonFactory.createObjectNode();
                        result.put("entityType", entity.getType());
                        result.put("id", entity.getId());
                        result.put("context", node.get("context"));
                        arrayNode.add(result);
                    }
                }
                return arrayNode;
            });
        }
        return entitiesField.map(entitiesArray -> {
            ObjectNode payload = objectNode.get("payload").toObjectNode();
            payload.set("entities", entitiesArray);
            return objectNode;
        }).timeout(Duration.ofSeconds(_timeout), Mono.error(new TimeoutException("timeout on entities count: " + entities.size() + " (Did not observe any item or terminal signal within " + Duration.ofSeconds(_timeout).toMillis() + "ms)")));
    });
}

Example 2 with Payload

use of io.openk9.entity.manager.dto.Payload in project openk9 by smclab.

the class EntityManagerBus method run.

@SneakyThrows
public void run() {
    while (true) {
        Payload request = _entityManagerQueue.take();
        TransactionContext transactionContext = _hazelcastInstance.newTransactionContext();
        transactionContext.beginTransaction();
        try {
            TransactionalMap<EntityKey, Entity> entityTransactionalMap = transactionContext.getMap("entityMap");
            TransactionalMap<EntityRelationKey, EntityRelation> transactionalEntityRelationMap = transactionContext.getMap("entityRelationMap");
            TransactionalMultiMap<DocumentKey, String> documentEntityMap = transactionContext.getMultiMap("documentEntityMap");
            EntityManagerRequest payload = request.getPayload();
            _loggerAggregator.emitLog("process ingestionId", payload.getIngestionId());
            long tenantId = payload.getTenantId();
            String ingestionId = payload.getIngestionId();
            List<EntityRequest> entities = request.getEntities();
            Map<EntityKey, Entity> localEntityMap = new HashMap<>(entities.size());
            for (EntityRequest entityRequest : entities) {
                String name = entityRequest.getName();
                String type = entityRequest.getType();
                String cacheId = Long.toString(_entityFlakeId.newId());
                EntityKey key = EntityKey.of(tenantId, name, type, cacheId, ingestionId);
                Entity entity = new Entity(null, cacheId, tenantId, name, type, null, ingestionId, false, true, entityRequest.getContext());
                entityTransactionalMap.set(key, entity);
                localEntityMap.put(key, entity);
                for (EntityRequest entityRequest2 : entities) {
                    for (RelationRequest relation : entityRequest2.getRelations()) {
                        if (relation.getTo().equals(entityRequest.getTmpId())) {
                            relation.setTo(entity.getCacheId());
                        }
                    }
                }
            }
            for (EntityRequest entity : entities) {
                List<RelationRequest> relations = entity.getRelations();
                if (relations == null || relations.isEmpty()) {
                    continue;
                }
                Collection<Entity> values = localEntityMap.values();
                Entity current = values.stream().filter(e -> e.getName().equals(entity.getName()) && e.getType().equals(entity.getType())).findFirst().orElse(null);
                if (current == null) {
                    continue;
                }
                for (RelationRequest relation : relations) {
                    String to = relation.getTo();
                    String name = relation.getName();
                    for (Entity value : values) {
                        if (value.getCacheId().equals(to)) {
                            long entityRelationId = _entityRelationFlakeId.newId();
                            EntityRelation entityRelation = new EntityRelation(entityRelationId, current.getCacheId(), ingestionId, name, value.getCacheId());
                            transactionalEntityRelationMap.set(EntityRelationKey.of(entityRelationId, current.getCacheId(), ingestionId), entityRelation);
                        }
                    }
                }
            }
            if (!localEntityMap.isEmpty()) {
                DocumentKey key = DocumentKey.of(payload.getDatasourceId(), payload.getContentId(), tenantId);
                for (Entity value : localEntityMap.values()) {
                    documentEntityMap.put(key, value.getCacheId());
                }
            }
        } catch (Exception e) {
            _log.error(e.getMessage(), e);
            transactionContext.rollbackTransaction();
        } finally {
            transactionContext.commitTransaction();
        }
    }
}

Example 3 with Payload

use of io.openk9.entity.manager.dto.Payload in project openk9 by smclab.

the class EntityManagerConsumer method consume.

@Incoming("entity-manager-request")
@Outgoing("entity-manager-response")
@Blocking
public Message<JsonObject> consume(Object obj) throws InterruptedException {
    JsonObject jsonObject = obj instanceof JsonObject ? (JsonObject) obj : new JsonObject(new String((byte[]) obj));
    Payload payload = jsonObject.mapTo(Payload.class);
    _entityManagerQueue.offer(payload, 45, TimeUnit.SECONDS);
    String replyTo = payload.getReplyTo();
    return Message.of(jsonObject, Metadata.of(new OutgoingRabbitMQMetadata.Builder().withRoutingKey(replyTo).withTimestamp(ZonedDateTime.now()).build()));
}

Example 4 with Payload

use of io.openk9.entity.manager.dto.Payload in project openk9 by smclab.

the class InsertIndexWriter method _createDocWriterRequest.

private Mono<DocWriteRequest> _createDocWriterRequest(String indexName, ObjectNode enrichProcessorContext) {
    return Mono.defer(() -> {
        ObjectNode objectNode = enrichProcessorContext.get("payload").toObjectNode();
        String contentId = objectNode.get("contentId").asText();
        return _search.search(factory -> {
            SearchRequest searchRequest = new SearchRequest(indexName);
            MatchQueryBuilder matchQueryBuilder = QueryBuilders.matchQuery("contentId", contentId);
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.query(matchQueryBuilder);
            return searchRequest.source(searchSourceBuilder);
        }).onErrorReturn(SearchUtil.EMPTY_SEARCH_RESPONSE).filter(e -> e.getHits().getHits().length > 0).flatMapIterable(SearchResponse::getHits).next().map(e -> new UpdateRequest(indexName, e.getId()).doc(objectNode.toString(), XContentType.JSON)).cast(DocWriteRequest.class).switchIfEmpty(Mono.fromSupplier(() -> new IndexRequest(indexName).source(objectNode.toString(), XContentType.JSON)));
    });
}

Example 5 with Payload

use of io.openk9.entity.manager.dto.Payload in project openk9 by smclab.

the class EnrichPipelineProcessor method _mapToEnrichProcessorContext.

private ObjectNode _mapToEnrichProcessorContext(Tuple3<DatasourceContext, ObjectNode, PluginDriverDTO> t3) {
    List<String> dependencies = t3.getT1().getEnrichItems().stream().map(EnrichItem::getServiceName).collect(Collectors.toList());
    ObjectNode objectNode = _jsonFactory.createObjectNode();
    objectNode.put("pluginDriver", _jsonFactory.fromObjectToJsonNode(t3.getT3()));
    objectNode.put("dependencies", _jsonFactory.fromObjectToJsonNode(dependencies));
    objectNode.put("datasourceContext", _jsonFactory.fromObjectToJsonNode(t3.getT1()));
    objectNode.put("payload", t3.getT2());
    return objectNode;
}