Search in sources :

Example 1 with EntityGraphService

use of io.openk9.entity.manager.service.graph.EntityGraphService in project openk9 by smclab.

the class CreateEntitiesRunnable method _disambiguate.

private Optional<EntityGraph> _disambiguate(EntityGraphService entityGraphService, List<EntityIndex> candidates, List<EntityIndex> entityRequestList, Entity currentEntityRequest, String[] uniqueEntities, int minHops, int maxHops) {
    String currentEntityRequestType = currentEntityRequest.getType();
    List<EntityGraph> result = List.of();
    if (_containsValue(uniqueEntities, currentEntityRequestType)) {
        if (candidates.size() == 1) {
            if (_log.isDebugEnabled()) {
                _log.debug("disambiguating entity with type " + currentEntityRequestType);
            }
            EntityIndex candidate = candidates.get(0);
            EntityGraph entityGraph = EntityGraph.of(candidate.getId(), candidate.getGraphId(), candidate.getTenantId(), candidate.getName(), candidate.getType());
            result = List.of(entityGraph);
        } else if (candidates.size() > 1) {
            result = _getEntityGraphs(entityGraphService, entityRequestList, minHops, maxHops, currentEntityRequestType, result);
        }
    } else {
        if (!candidates.isEmpty()) {
            result = _getEntityGraphs(entityGraphService, entityRequestList, minHops, maxHops, currentEntityRequestType, result);
        }
    }
    if (_log.isDebugEnabled()) {
        _log.debug("_disambiguate: " + result + " current: " + currentEntityRequest);
    }
    return result.stream().filter(Objects::nonNull).filter(entityGraph -> candidates.stream().anyMatch(entityIndex -> entityGraph.getId().equals(entityIndex.getId()))).findFirst();
}
Also used : EntityGraph(io.openk9.entity.manager.model.graph.EntityGraph) Arrays(java.util.Arrays) Member(com.hazelcast.cluster.Member) FutureUtil(io.openk9.entity.manager.util.FutureUtil) EntityKey(io.openk9.entity.manager.cache.model.EntityKey) SymbolicName(org.neo4j.cypherdsl.core.SymbolicName) Future(java.util.concurrent.Future) Map(java.util.Map) ToString(lombok.ToString) Property(org.neo4j.cypherdsl.core.Property) Cypher(org.neo4j.cypherdsl.core.Cypher) EntityIndex(io.openk9.entity.manager.model.index.EntityIndex) EntityGraph(io.openk9.entity.manager.model.graph.EntityGraph) Collection(java.util.Collection) CDI(javax.enterprise.inject.spi.CDI) Set(java.util.Set) Collectors(java.util.stream.Collectors) Serializable(java.io.Serializable) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) Builder(lombok.Builder) Optional(java.util.Optional) Cypher.literalOf(org.neo4j.cypherdsl.core.Cypher.literalOf) HazelcastInstanceAware(com.hazelcast.core.HazelcastInstanceAware) AssociableEntityKey(io.openk9.entity.manager.cache.model.AssociableEntityKey) EntityGraphConfig(io.openk9.entity.manager.config.EntityGraphConfig) Logger(org.jboss.logging.Logger) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Statement(org.neo4j.cypherdsl.core.Statement) Schedulers(reactor.core.scheduler.Schedulers) EntityNameCleanerProvider(io.openk9.entity.manager.cleaner.EntityNameCleanerProvider) Node(org.neo4j.cypherdsl.core.Node) HazelcastInstance(com.hazelcast.core.HazelcastInstance) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) EntityNameCleaner(io.openk9.entity.manager.cleaner.EntityNameCleaner) Entity(io.openk9.entity.manager.cache.model.Entity) Functions(org.neo4j.cypherdsl.core.Functions) Mono(reactor.core.publisher.Mono) IExecutorService(com.hazelcast.core.IExecutorService) GetEntitiesCallable(io.openk9.entity.manager.action.GetEntitiesCallable) AliasedExpression(org.neo4j.cypherdsl.core.AliasedExpression) Predicates(com.hazelcast.query.Predicates) EntityService(io.openk9.entity.manager.service.index.EntityService) Data(lombok.Data) AllArgsConstructor(lombok.AllArgsConstructor) EntityGraphService(io.openk9.entity.manager.service.graph.EntityGraphService) Collections(java.util.Collections) MapUtil(io.openk9.entity.manager.util.MapUtil) IMap(com.hazelcast.map.IMap) Objects(java.util.Objects) EntityIndex(io.openk9.entity.manager.model.index.EntityIndex) ToString(lombok.ToString)

Example 2 with EntityGraphService

use of io.openk9.entity.manager.service.graph.EntityGraphService in project openk9 by smclab.

the class CreateEntitiesRunnable method _getEntityGraphs.

private List<EntityGraph> _getEntityGraphs(EntityGraphService entityGraphService, List<EntityIndex> entityRequestList, int minHops, int maxHops, String currentEntityRequestType, List<EntityGraph> result) {
    if (_log.isDebugEnabled()) {
        _log.debug("disambiguating with search entity with type " + currentEntityRequestType);
    }
    Statement[] statements = new Statement[entityRequestList.size()];
    for (int i = 0; i < entityRequestList.size(); i++) {
        EntityIndex entityRequest = entityRequestList.get(i);
        Node nodeEntity = Cypher.node(entityRequest.getType()).named("entity");
        AliasedExpression entityAliased = nodeEntity.as("entity");
        SymbolicName path = Cypher.name("path");
        Property idProperty = entityAliased.getDelegate().property("id");
        Statement statement = Cypher.match(nodeEntity).where(idProperty.eq(literalOf(entityRequest.getId()))).call("apoc.path.expand").withArgs(entityAliased.getDelegate(), literalOf(null), literalOf("-date"), literalOf(minHops), literalOf(maxHops)).yield(path).returning(Functions.last(Functions.nodes(path)).as("node"), Functions.size(Functions.nodes(path)).subtract(literalOf(1)).as("hops")).build();
        statements[i] = statement;
    }
    if (statements.length == 1) {
        Statement entityRequestListStatement = Cypher.call(statements[0]).returning("node", "hops").orderBy(Cypher.name("hops")).build();
        result = entityGraphService.search(entityRequestListStatement);
    } else if (statements.length > 1) {
        Statement entityRequestListStatement = Cypher.call(Cypher.unionAll(statements)).returning("node", "hops").orderBy(Cypher.name("hops")).build();
        result = entityGraphService.search(entityRequestListStatement);
    }
    return result;
}
Also used : Statement(org.neo4j.cypherdsl.core.Statement) Node(org.neo4j.cypherdsl.core.Node) EntityIndex(io.openk9.entity.manager.model.index.EntityIndex) SymbolicName(org.neo4j.cypherdsl.core.SymbolicName) Property(org.neo4j.cypherdsl.core.Property) AliasedExpression(org.neo4j.cypherdsl.core.AliasedExpression)

Example 3 with EntityGraphService

use of io.openk9.entity.manager.service.graph.EntityGraphService in project openk9 by smclab.

the class CreateDocumentEntitiesRunnable method run_.

@Override
public void run_() {
    _log.info("start CreateEntitiesRunnable");
    IMap<EntityKey, Entity> entityIMap = MapUtil.getEntityMap(_hazelcastInstance);
    MultiMap<DocumentKey, String> documentEntityMapMap = MapUtil.getDocumentEntityMapMap(_hazelcastInstance);
    EntityGraphService entityGraphService = CDI.current().select(EntityGraphService.class).get();
    for (DocumentKey documentKey : documentEntityMapMap.localKeySet()) {
        Collection<String> entityCacheIds = documentEntityMapMap.get(documentKey);
        String[] arr = entityCacheIds.toArray(String[]::new);
        Collection<Object> project = entityIMap.project(Projections.singleAttribute("indexable"), Predicates.and(Predicates.in("cacheId", arr), Predicates.notEqual("id", null), Predicates.notEqual("graphId", null)));
        if (project.size() == entityCacheIds.size()) {
            Collection<Entity> entities = entityIMap.values(Predicates.in("cacheId", arr));
            DocumentGraph document = entityGraphService.insertDocument(DocumentGraph.of(null, documentKey.getDatasourceId(), documentKey.getTenantId(), documentKey.getContentId()));
            for (Entity entity : entities) {
                entityGraphService.createDocumentRelationship(entity.getId(), document.getId(), "related_to");
            }
            documentEntityMapMap.remove(documentKey);
        }
    }
}
Also used : Entity(io.openk9.entity.manager.cache.model.Entity) EntityGraphService(io.openk9.entity.manager.service.graph.EntityGraphService) EntityKey(io.openk9.entity.manager.cache.model.EntityKey) DocumentKey(io.openk9.entity.manager.cache.model.DocumentKey) DocumentGraph(io.openk9.entity.manager.model.graph.DocumentGraph)

Example 4 with EntityGraphService

use of io.openk9.entity.manager.service.graph.EntityGraphService in project openk9 by smclab.

the class CreateEntitiesRunnable method run_.

@Override
public void run_() {
    _log.info("start CreateEntitiesRunnable");
    IMap<EntityKey, Entity> entityIMap = MapUtil.getEntityMap(_hazelcastInstance);
    IMap<AssociableEntityKey, Entity> associableEntityMap = MapUtil.getAssociableEntityMap(_hazelcastInstance);
    Set<EntityKey> entityKeys = entityIMap.localKeySet(Predicates.and(Predicates.equal("id", null), Predicates.equal("graphId", null)));
    EntityGraphConfig config = CDI.current().select(EntityGraphConfig.class).get();
    EntityNameCleanerProvider entityNameCleanerProvider = CDI.current().select(EntityNameCleanerProvider.class).get();
    EntityService entityService = CDI.current().select(EntityService.class).get();
    EntityGraphService entityGraphService = CDI.current().select(EntityGraphService.class).get();
    Map<EntityKey, Entity> localEntityMap = entityIMap.getAll(entityKeys);
    Collection<Entity> localEntityValues = localEntityMap.values();
    Set<EntityKey> localEntityKeys = localEntityMap.keySet();
    List<Member> collect = _hazelcastInstance.getCluster().getMembers().stream().filter(member -> !member.localMember()).collect(Collectors.toList());
    String[] ingestionIds = localEntityKeys.stream().map(EntityKey::getIngestionId).distinct().toArray(String[]::new);
    IExecutorService entityExecutor = _hazelcastInstance.getExecutorService("entityExecutor");
    Map<Member, Future<Map<EntityKey, Entity>>> memberFutureMap = entityExecutor.submitToMembers(new GetEntitiesCallable(ingestionIds), collect);
    Map<EntityKey, Entity> otherEntityKeyEntityMap = memberFutureMap.values().stream().map(FutureUtil::makeCompletableFuture).map(CompletableFuture::join).reduce((a, b) -> {
        Map<EntityKey, Entity> map = new HashMap<>();
        map.putAll(a);
        map.putAll(b);
        return map;
    }).orElseGet(Map::of);
    Stream<EntityMember> otherEntityMemberStream = otherEntityKeyEntityMap.values().stream().map(entity -> EntityMember.of(entity, false));
    Stream<EntityMember> localEntityMemberStream = localEntityValues.stream().map(entity -> EntityMember.of(entity, true));
    Map<String, List<EntityMember>> entitiesGroupingByIngestionId = Stream.concat(localEntityMemberStream, otherEntityMemberStream).collect(Collectors.groupingBy(entityMember -> entityMember.getEntity().getIngestionId()));
    Collection<List<EntityMember>> values = entitiesGroupingByIngestionId.values();
    Map<EntityKey, Entity> entityMap = new HashMap<>();
    for (List<EntityMember> ingestionIdEntities : values) {
        Map<AssociableEntityKey, Entity> localAssociableEntityMap = new HashMap<>();
        List<EntityCandidates> entityCandidateList = new ArrayList<>();
        for (EntityMember ingestionIdEntity : ingestionIdEntities) {
            Entity innerEntity = ingestionIdEntity.getEntity();
            entityCandidateList.add(getEntityCandidates(entityNameCleanerProvider, entityService, ingestionIdEntity, innerEntity));
        }
        List<Mono<Entity>> completableFutureList = entityCandidateList.stream().filter(entityCandidates -> entityCandidates.getEntity().isLocal()).map(entityCandidates -> Mono.fromSupplier(_getAndCreateEntityDisambiguate(config, entityNameCleanerProvider, entityService, entityGraphService, entityCandidateList, entityCandidates, entityCandidates.getEntity())).subscribeOn(Schedulers.boundedElastic())).collect(Collectors.toList());
        Mono<List<Entity>> zip = Mono.zip(completableFutureList, a -> {
            List<Entity> entities = new ArrayList<>();
            for (Object o : a) {
                entities.add((Entity) o);
            }
            return entities;
        }).defaultIfEmpty(List.of());
        for (Entity currentEntityRequest : zip.block()) {
            localAssociableEntityMap.put(AssociableEntityKey.of(currentEntityRequest.getCacheId(), currentEntityRequest.getIngestionId()), currentEntityRequest);
            entityMap.put(EntityKey.of(currentEntityRequest.getTenantId(), currentEntityRequest.getName(), currentEntityRequest.getType(), currentEntityRequest.getCacheId(), currentEntityRequest.getIngestionId()), currentEntityRequest);
        }
        associableEntityMap.setAll(localAssociableEntityMap);
    }
    entityIMap.setAll(entityMap);
}
Also used : Arrays(java.util.Arrays) Member(com.hazelcast.cluster.Member) FutureUtil(io.openk9.entity.manager.util.FutureUtil) EntityKey(io.openk9.entity.manager.cache.model.EntityKey) SymbolicName(org.neo4j.cypherdsl.core.SymbolicName) Future(java.util.concurrent.Future) Map(java.util.Map) ToString(lombok.ToString) Property(org.neo4j.cypherdsl.core.Property) Cypher(org.neo4j.cypherdsl.core.Cypher) EntityIndex(io.openk9.entity.manager.model.index.EntityIndex) EntityGraph(io.openk9.entity.manager.model.graph.EntityGraph) Collection(java.util.Collection) CDI(javax.enterprise.inject.spi.CDI) Set(java.util.Set) Collectors(java.util.stream.Collectors) Serializable(java.io.Serializable) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) Builder(lombok.Builder) Optional(java.util.Optional) Cypher.literalOf(org.neo4j.cypherdsl.core.Cypher.literalOf) HazelcastInstanceAware(com.hazelcast.core.HazelcastInstanceAware) AssociableEntityKey(io.openk9.entity.manager.cache.model.AssociableEntityKey) EntityGraphConfig(io.openk9.entity.manager.config.EntityGraphConfig) Logger(org.jboss.logging.Logger) HashMap(java.util.HashMap) CompletableFuture(java.util.concurrent.CompletableFuture) Supplier(java.util.function.Supplier) ArrayList(java.util.ArrayList) Statement(org.neo4j.cypherdsl.core.Statement) Schedulers(reactor.core.scheduler.Schedulers) EntityNameCleanerProvider(io.openk9.entity.manager.cleaner.EntityNameCleanerProvider) Node(org.neo4j.cypherdsl.core.Node) HazelcastInstance(com.hazelcast.core.HazelcastInstance) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) EntityNameCleaner(io.openk9.entity.manager.cleaner.EntityNameCleaner) Entity(io.openk9.entity.manager.cache.model.Entity) Functions(org.neo4j.cypherdsl.core.Functions) Mono(reactor.core.publisher.Mono) IExecutorService(com.hazelcast.core.IExecutorService) GetEntitiesCallable(io.openk9.entity.manager.action.GetEntitiesCallable) AliasedExpression(org.neo4j.cypherdsl.core.AliasedExpression) Predicates(com.hazelcast.query.Predicates) EntityService(io.openk9.entity.manager.service.index.EntityService) Data(lombok.Data) AllArgsConstructor(lombok.AllArgsConstructor) EntityGraphService(io.openk9.entity.manager.service.graph.EntityGraphService) Collections(java.util.Collections) MapUtil(io.openk9.entity.manager.util.MapUtil) IMap(com.hazelcast.map.IMap) Entity(io.openk9.entity.manager.cache.model.Entity) EntityGraphService(io.openk9.entity.manager.service.graph.EntityGraphService) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) ToString(lombok.ToString) IExecutorService(com.hazelcast.core.IExecutorService) EntityKey(io.openk9.entity.manager.cache.model.EntityKey) AssociableEntityKey(io.openk9.entity.manager.cache.model.AssociableEntityKey) CompletableFuture(java.util.concurrent.CompletableFuture) GetEntitiesCallable(io.openk9.entity.manager.action.GetEntitiesCallable) List(java.util.List) ArrayList(java.util.ArrayList) EntityGraphConfig(io.openk9.entity.manager.config.EntityGraphConfig) Member(com.hazelcast.cluster.Member) EntityService(io.openk9.entity.manager.service.index.EntityService) Mono(reactor.core.publisher.Mono) EntityNameCleanerProvider(io.openk9.entity.manager.cleaner.EntityNameCleanerProvider) Future(java.util.concurrent.Future) CompletableFuture(java.util.concurrent.CompletableFuture) AssociableEntityKey(io.openk9.entity.manager.cache.model.AssociableEntityKey) Map(java.util.Map) HashMap(java.util.HashMap) IMap(com.hazelcast.map.IMap)

Example 5 with EntityGraphService

use of io.openk9.entity.manager.service.graph.EntityGraphService in project openk9 by smclab.

the class CreateRelationRunnable method run_.

@Override
public void run_() {
    IMap<EntityKey, Entity> entityIMap = MapUtil.getEntityMap(_hazelcastInstance);
    Set<EntityKey> entityKeys = entityIMap.localKeySet(Predicates.and(Predicates.notEqual("id", null), Predicates.notEqual("graphId", null)));
    _log.info("entityKeys: " + entityKeys.size());
    Map<EntityKey, Entity> entityIMapAll = entityIMap.getAll(entityKeys);
    _log.info("entityIMapAll: " + entityIMapAll.size());
    IMap<EntityRelationKey, EntityRelation> entityRelationMap = MapUtil.getEntityRelationMap(_hazelcastInstance);
    Map<String, String> collect = entityIMapAll.values().stream().collect(Collectors.toMap(Entity::getCacheId, Entity::getId));
    String[] cacheIds = entityKeys.stream().map(EntityKey::getCacheId).distinct().toArray(String[]::new);
    Map<EntityRelationKey, EntityRelation> entries = entityRelationMap.getAll(entityRelationMap.keySet(Predicates.in("__key.entityId", cacheIds)));
    _log.info("entityRelations: " + entries.size());
    EntityGraphService entityGraphService = CDI.current().select(EntityGraphService.class).get();
    List<EntityRelationKey> entityRelationKeysToDelete = new ArrayList<>();
    for (Map.Entry<EntityRelationKey, EntityRelation> entry : entries.entrySet()) {
        EntityRelationKey key = entry.getKey();
        EntityRelation value = entry.getValue();
        String from = collect.get(value.getEntityCacheId());
        String to = collect.get(value.getTo());
        if (from != null && to != null) {
            try {
                entityGraphService.createRelationship(from, to, value.getName());
                entityRelationKeysToDelete.add(key);
            } catch (Exception e) {
                _log.error(e.getMessage(), e);
            }
        }
    }
    try {
        Pipelining pipelining = new Pipelining<>(10);
        for (EntityRelationKey entityRelationKey : entityRelationKeysToDelete) {
            pipelining.add(entityRelationMap.removeAsync(entityRelationKey));
        }
        pipelining.results();
    } catch (Exception e) {
        _log.error(e.getMessage(), e);
    }
}
Also used : Entity(io.openk9.entity.manager.cache.model.Entity) EntityGraphService(io.openk9.entity.manager.service.graph.EntityGraphService) ArrayList(java.util.ArrayList) EntityRelationKey(io.openk9.entity.manager.cache.model.EntityRelationKey) EntityKey(io.openk9.entity.manager.cache.model.EntityKey) EntityRelation(io.openk9.entity.manager.cache.model.EntityRelation) Pipelining(com.hazelcast.core.Pipelining) Map(java.util.Map) IMap(com.hazelcast.map.IMap)

Aggregations

Entity (io.openk9.entity.manager.cache.model.Entity)4 EntityKey (io.openk9.entity.manager.cache.model.EntityKey)4 EntityGraphService (io.openk9.entity.manager.service.graph.EntityGraphService)4 IMap (com.hazelcast.map.IMap)3 EntityIndex (io.openk9.entity.manager.model.index.EntityIndex)3 ArrayList (java.util.ArrayList)3 Map (java.util.Map)3 Member (com.hazelcast.cluster.Member)2 HazelcastInstance (com.hazelcast.core.HazelcastInstance)2 HazelcastInstanceAware (com.hazelcast.core.HazelcastInstanceAware)2 IExecutorService (com.hazelcast.core.IExecutorService)2 Predicates (com.hazelcast.query.Predicates)2 GetEntitiesCallable (io.openk9.entity.manager.action.GetEntitiesCallable)2 AssociableEntityKey (io.openk9.entity.manager.cache.model.AssociableEntityKey)2 EntityNameCleaner (io.openk9.entity.manager.cleaner.EntityNameCleaner)2 EntityNameCleanerProvider (io.openk9.entity.manager.cleaner.EntityNameCleanerProvider)2 EntityGraphConfig (io.openk9.entity.manager.config.EntityGraphConfig)2 EntityGraph (io.openk9.entity.manager.model.graph.EntityGraph)2 EntityService (io.openk9.entity.manager.service.index.EntityService)2 FutureUtil (io.openk9.entity.manager.util.FutureUtil)2