Examples with Entity io.openk9.entity.manager.cache.model.Entity used on opensource projects

Example 11 with Entity

use of io.openk9.entity.manager.cache.model.Entity in project openk9 by smclab.

the class CreateEntitiesRunnable method run_.

@Override
public void run_() {
    _log.info("start CreateEntitiesRunnable");
    IMap<EntityKey, Entity> entityIMap = MapUtil.getEntityMap(_hazelcastInstance);
    IMap<AssociableEntityKey, Entity> associableEntityMap = MapUtil.getAssociableEntityMap(_hazelcastInstance);
    Set<EntityKey> entityKeys = entityIMap.localKeySet(Predicates.and(Predicates.equal("id", null), Predicates.equal("graphId", null)));
    EntityGraphConfig config = CDI.current().select(EntityGraphConfig.class).get();
    EntityNameCleanerProvider entityNameCleanerProvider = CDI.current().select(EntityNameCleanerProvider.class).get();
    EntityService entityService = CDI.current().select(EntityService.class).get();
    EntityGraphService entityGraphService = CDI.current().select(EntityGraphService.class).get();
    Map<EntityKey, Entity> localEntityMap = entityIMap.getAll(entityKeys);
    Collection<Entity> localEntityValues = localEntityMap.values();
    Set<EntityKey> localEntityKeys = localEntityMap.keySet();
    List<Member> collect = _hazelcastInstance.getCluster().getMembers().stream().filter(member -> !member.localMember()).collect(Collectors.toList());
    String[] ingestionIds = localEntityKeys.stream().map(EntityKey::getIngestionId).distinct().toArray(String[]::new);
    IExecutorService entityExecutor = _hazelcastInstance.getExecutorService("entityExecutor");
    Map<Member, Future<Map<EntityKey, Entity>>> memberFutureMap = entityExecutor.submitToMembers(new GetEntitiesCallable(ingestionIds), collect);
    Map<EntityKey, Entity> otherEntityKeyEntityMap = memberFutureMap.values().stream().map(FutureUtil::makeCompletableFuture).map(CompletableFuture::join).reduce((a, b) -> {
        Map<EntityKey, Entity> map = new HashMap<>();
        map.putAll(a);
        map.putAll(b);
        return map;
    }).orElseGet(Map::of);
    Stream<EntityMember> otherEntityMemberStream = otherEntityKeyEntityMap.values().stream().map(entity -> EntityMember.of(entity, false));
    Stream<EntityMember> localEntityMemberStream = localEntityValues.stream().map(entity -> EntityMember.of(entity, true));
    Map<String, List<EntityMember>> entitiesGroupingByIngestionId = Stream.concat(localEntityMemberStream, otherEntityMemberStream).collect(Collectors.groupingBy(entityMember -> entityMember.getEntity().getIngestionId()));
    Collection<List<EntityMember>> values = entitiesGroupingByIngestionId.values();
    Map<EntityKey, Entity> entityMap = new HashMap<>();
    for (List<EntityMember> ingestionIdEntities : values) {
        Map<AssociableEntityKey, Entity> localAssociableEntityMap = new HashMap<>();
        List<EntityCandidates> entityCandidateList = new ArrayList<>();
        for (EntityMember ingestionIdEntity : ingestionIdEntities) {
            Entity innerEntity = ingestionIdEntity.getEntity();
            entityCandidateList.add(getEntityCandidates(entityNameCleanerProvider, entityService, ingestionIdEntity, innerEntity));
        }
        List<Mono<Entity>> completableFutureList = entityCandidateList.stream().filter(entityCandidates -> entityCandidates.getEntity().isLocal()).map(entityCandidates -> Mono.fromSupplier(_getAndCreateEntityDisambiguate(config, entityNameCleanerProvider, entityService, entityGraphService, entityCandidateList, entityCandidates, entityCandidates.getEntity())).subscribeOn(Schedulers.boundedElastic())).collect(Collectors.toList());
        Mono<List<Entity>> zip = Mono.zip(completableFutureList, a -> {
            List<Entity> entities = new ArrayList<>();
            for (Object o : a) {
                entities.add((Entity) o);
            }
            return entities;
        }).defaultIfEmpty(List.of());
        for (Entity currentEntityRequest : zip.block()) {
            localAssociableEntityMap.put(AssociableEntityKey.of(currentEntityRequest.getCacheId(), currentEntityRequest.getIngestionId()), currentEntityRequest);
            entityMap.put(EntityKey.of(currentEntityRequest.getTenantId(), currentEntityRequest.getName(), currentEntityRequest.getType(), currentEntityRequest.getCacheId(), currentEntityRequest.getIngestionId()), currentEntityRequest);
        }
        associableEntityMap.setAll(localAssociableEntityMap);
    }
    entityIMap.setAll(entityMap);
}

Example 12 with Entity

use of io.openk9.entity.manager.cache.model.Entity in project openk9 by smclab.

the class CreateEntitiesRunnable method cleanCandidates.

private List<EntityIndex> cleanCandidates(Entity entityRequest, List<EntityIndex> candidates, EntityNameCleanerProvider entityNameCleanerProvider, float scoreThreshold) {
    if (!candidates.isEmpty()) {
        EntityIndex documentEntityResponse = candidates.get(0);
        double bestScore;
        if (candidates.size() > 1) {
            double[] scores = candidates.stream().mapToDouble(EntityIndex::getScore).toArray();
            bestScore = _softmax(documentEntityResponse.getScore(), scores);
        } else {
            bestScore = _levenshteinDistance(entityNameCleanerProvider.get(documentEntityResponse.getType()).cleanEntityName(documentEntityResponse.getName()), entityNameCleanerProvider.get(entityRequest.getType()).cleanEntityName(entityRequest.getName()));
        }
        if (bestScore > scoreThreshold) {
            return Collections.singletonList(documentEntityResponse);
        }
    }
    return candidates;
}

Example 13 with Entity

use of io.openk9.entity.manager.cache.model.Entity in project openk9 by smclab.

the class CreateRelationRunnable method run_.

@Override
public void run_() {
    IMap<EntityKey, Entity> entityIMap = MapUtil.getEntityMap(_hazelcastInstance);
    Set<EntityKey> entityKeys = entityIMap.localKeySet(Predicates.and(Predicates.notEqual("id", null), Predicates.notEqual("graphId", null)));
    _log.info("entityKeys: " + entityKeys.size());
    Map<EntityKey, Entity> entityIMapAll = entityIMap.getAll(entityKeys);
    _log.info("entityIMapAll: " + entityIMapAll.size());
    IMap<EntityRelationKey, EntityRelation> entityRelationMap = MapUtil.getEntityRelationMap(_hazelcastInstance);
    Map<String, String> collect = entityIMapAll.values().stream().collect(Collectors.toMap(Entity::getCacheId, Entity::getId));
    String[] cacheIds = entityKeys.stream().map(EntityKey::getCacheId).distinct().toArray(String[]::new);
    Map<EntityRelationKey, EntityRelation> entries = entityRelationMap.getAll(entityRelationMap.keySet(Predicates.in("__key.entityId", cacheIds)));
    _log.info("entityRelations: " + entries.size());
    EntityGraphService entityGraphService = CDI.current().select(EntityGraphService.class).get();
    List<EntityRelationKey> entityRelationKeysToDelete = new ArrayList<>();
    for (Map.Entry<EntityRelationKey, EntityRelation> entry : entries.entrySet()) {
        EntityRelationKey key = entry.getKey();
        EntityRelation value = entry.getValue();
        String from = collect.get(value.getEntityCacheId());
        String to = collect.get(value.getTo());
        if (from != null && to != null) {
            try {
                entityGraphService.createRelationship(from, to, value.getName());
                entityRelationKeysToDelete.add(key);
            } catch (Exception e) {
                _log.error(e.getMessage(), e);
            }
        }
    }
    try {
        Pipelining pipelining = new Pipelining<>(10);
        for (EntityRelationKey entityRelationKey : entityRelationKeysToDelete) {
            pipelining.add(entityRelationMap.removeAsync(entityRelationKey));
        }
        pipelining.results();
    } catch (Exception e) {
        _log.error(e.getMessage(), e);
    }
}

Example 14 with Entity

use of io.openk9.entity.manager.cache.model.Entity in project openk9 by smclab.

the class EntityManagerConsumer method consume.

@Incoming("entity-manager-request")
@Outgoing("entity-manager-response")
@Blocking
public Message<JsonObject> consume(Object obj) throws InterruptedException {
    JsonObject jsonObject = obj instanceof JsonObject ? (JsonObject) obj : new JsonObject(new String((byte[]) obj));
    Payload payload = jsonObject.mapTo(Payload.class);
    _entityManagerQueue.offer(payload, 45, TimeUnit.SECONDS);
    String replyTo = payload.getReplyTo();
    return Message.of(jsonObject, Metadata.of(new OutgoingRabbitMQMetadata.Builder().withRoutingKey(replyTo).withTimestamp(ZonedDateTime.now()).build()));
}

Example 15 with Entity

use of io.openk9.entity.manager.cache.model.Entity in project openk9 by smclab.

the class EntityService method search.

public List<EntityIndex> search(long tenantId, QueryBuilder queryBuilder, int from, int size) {
    SearchRequest searchRequest = new SearchRequest(tenantId + "-entity");
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.query(queryBuilder);
    searchSourceBuilder.size(from);
    searchSourceBuilder.size(size);
    searchRequest.source(searchSourceBuilder);
    try {
        SearchResponse searchResponse = _restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        SearchHits hits = searchResponse.getHits();
        List<EntityIndex> results = new ArrayList<>(hits.getHits().length);
        for (SearchHit hit : hits.getHits()) {
            String sourceAsString = hit.getSourceAsString();
            JsonObject json = new JsonObject(sourceAsString);
            EntityIndex entityIndex = json.mapTo(EntityIndex.class);
            entityIndex.setScore(hit.getScore());
            results.add(entityIndex);
        }
        return results;
    } catch (Exception e) {
        _logger.error(e.getMessage());
    }
    return List.of();
}