use of io.openk9.entity.manager.service.graph.EntityGraphService in project openk9 by smclab.
the class CreateEntitiesRunnable method _disambiguate.
private Optional<EntityGraph> _disambiguate(EntityGraphService entityGraphService, List<EntityIndex> candidates, List<EntityIndex> entityRequestList, Entity currentEntityRequest, String[] uniqueEntities, int minHops, int maxHops) {
String currentEntityRequestType = currentEntityRequest.getType();
List<EntityGraph> result = List.of();
if (_containsValue(uniqueEntities, currentEntityRequestType)) {
if (candidates.size() == 1) {
if (_log.isDebugEnabled()) {
_log.debug("disambiguating entity with type " + currentEntityRequestType);
}
EntityIndex candidate = candidates.get(0);
EntityGraph entityGraph = EntityGraph.of(candidate.getId(), candidate.getGraphId(), candidate.getTenantId(), candidate.getName(), candidate.getType());
result = List.of(entityGraph);
} else if (candidates.size() > 1) {
result = _getEntityGraphs(entityGraphService, entityRequestList, minHops, maxHops, currentEntityRequestType, result);
}
} else {
if (!candidates.isEmpty()) {
result = _getEntityGraphs(entityGraphService, entityRequestList, minHops, maxHops, currentEntityRequestType, result);
}
}
if (_log.isDebugEnabled()) {
_log.debug("_disambiguate: " + result + " current: " + currentEntityRequest);
}
return result.stream().filter(Objects::nonNull).filter(entityGraph -> candidates.stream().anyMatch(entityIndex -> entityGraph.getId().equals(entityIndex.getId()))).findFirst();
}
use of io.openk9.entity.manager.service.graph.EntityGraphService in project openk9 by smclab.
the class CreateEntitiesRunnable method _getEntityGraphs.
private List<EntityGraph> _getEntityGraphs(EntityGraphService entityGraphService, List<EntityIndex> entityRequestList, int minHops, int maxHops, String currentEntityRequestType, List<EntityGraph> result) {
if (_log.isDebugEnabled()) {
_log.debug("disambiguating with search entity with type " + currentEntityRequestType);
}
Statement[] statements = new Statement[entityRequestList.size()];
for (int i = 0; i < entityRequestList.size(); i++) {
EntityIndex entityRequest = entityRequestList.get(i);
Node nodeEntity = Cypher.node(entityRequest.getType()).named("entity");
AliasedExpression entityAliased = nodeEntity.as("entity");
SymbolicName path = Cypher.name("path");
Property idProperty = entityAliased.getDelegate().property("id");
Statement statement = Cypher.match(nodeEntity).where(idProperty.eq(literalOf(entityRequest.getId()))).call("apoc.path.expand").withArgs(entityAliased.getDelegate(), literalOf(null), literalOf("-date"), literalOf(minHops), literalOf(maxHops)).yield(path).returning(Functions.last(Functions.nodes(path)).as("node"), Functions.size(Functions.nodes(path)).subtract(literalOf(1)).as("hops")).build();
statements[i] = statement;
}
if (statements.length == 1) {
Statement entityRequestListStatement = Cypher.call(statements[0]).returning("node", "hops").orderBy(Cypher.name("hops")).build();
result = entityGraphService.search(entityRequestListStatement);
} else if (statements.length > 1) {
Statement entityRequestListStatement = Cypher.call(Cypher.unionAll(statements)).returning("node", "hops").orderBy(Cypher.name("hops")).build();
result = entityGraphService.search(entityRequestListStatement);
}
return result;
}
use of io.openk9.entity.manager.service.graph.EntityGraphService in project openk9 by smclab.
the class CreateDocumentEntitiesRunnable method run_.
@Override
public void run_() {
_log.info("start CreateEntitiesRunnable");
IMap<EntityKey, Entity> entityIMap = MapUtil.getEntityMap(_hazelcastInstance);
MultiMap<DocumentKey, String> documentEntityMapMap = MapUtil.getDocumentEntityMapMap(_hazelcastInstance);
EntityGraphService entityGraphService = CDI.current().select(EntityGraphService.class).get();
for (DocumentKey documentKey : documentEntityMapMap.localKeySet()) {
Collection<String> entityCacheIds = documentEntityMapMap.get(documentKey);
String[] arr = entityCacheIds.toArray(String[]::new);
Collection<Object> project = entityIMap.project(Projections.singleAttribute("indexable"), Predicates.and(Predicates.in("cacheId", arr), Predicates.notEqual("id", null), Predicates.notEqual("graphId", null)));
if (project.size() == entityCacheIds.size()) {
Collection<Entity> entities = entityIMap.values(Predicates.in("cacheId", arr));
DocumentGraph document = entityGraphService.insertDocument(DocumentGraph.of(null, documentKey.getDatasourceId(), documentKey.getTenantId(), documentKey.getContentId()));
for (Entity entity : entities) {
entityGraphService.createDocumentRelationship(entity.getId(), document.getId(), "related_to");
}
documentEntityMapMap.remove(documentKey);
}
}
}
use of io.openk9.entity.manager.service.graph.EntityGraphService in project openk9 by smclab.
the class CreateEntitiesRunnable method run_.
@Override
public void run_() {
_log.info("start CreateEntitiesRunnable");
IMap<EntityKey, Entity> entityIMap = MapUtil.getEntityMap(_hazelcastInstance);
IMap<AssociableEntityKey, Entity> associableEntityMap = MapUtil.getAssociableEntityMap(_hazelcastInstance);
Set<EntityKey> entityKeys = entityIMap.localKeySet(Predicates.and(Predicates.equal("id", null), Predicates.equal("graphId", null)));
EntityGraphConfig config = CDI.current().select(EntityGraphConfig.class).get();
EntityNameCleanerProvider entityNameCleanerProvider = CDI.current().select(EntityNameCleanerProvider.class).get();
EntityService entityService = CDI.current().select(EntityService.class).get();
EntityGraphService entityGraphService = CDI.current().select(EntityGraphService.class).get();
Map<EntityKey, Entity> localEntityMap = entityIMap.getAll(entityKeys);
Collection<Entity> localEntityValues = localEntityMap.values();
Set<EntityKey> localEntityKeys = localEntityMap.keySet();
List<Member> collect = _hazelcastInstance.getCluster().getMembers().stream().filter(member -> !member.localMember()).collect(Collectors.toList());
String[] ingestionIds = localEntityKeys.stream().map(EntityKey::getIngestionId).distinct().toArray(String[]::new);
IExecutorService entityExecutor = _hazelcastInstance.getExecutorService("entityExecutor");
Map<Member, Future<Map<EntityKey, Entity>>> memberFutureMap = entityExecutor.submitToMembers(new GetEntitiesCallable(ingestionIds), collect);
Map<EntityKey, Entity> otherEntityKeyEntityMap = memberFutureMap.values().stream().map(FutureUtil::makeCompletableFuture).map(CompletableFuture::join).reduce((a, b) -> {
Map<EntityKey, Entity> map = new HashMap<>();
map.putAll(a);
map.putAll(b);
return map;
}).orElseGet(Map::of);
Stream<EntityMember> otherEntityMemberStream = otherEntityKeyEntityMap.values().stream().map(entity -> EntityMember.of(entity, false));
Stream<EntityMember> localEntityMemberStream = localEntityValues.stream().map(entity -> EntityMember.of(entity, true));
Map<String, List<EntityMember>> entitiesGroupingByIngestionId = Stream.concat(localEntityMemberStream, otherEntityMemberStream).collect(Collectors.groupingBy(entityMember -> entityMember.getEntity().getIngestionId()));
Collection<List<EntityMember>> values = entitiesGroupingByIngestionId.values();
Map<EntityKey, Entity> entityMap = new HashMap<>();
for (List<EntityMember> ingestionIdEntities : values) {
Map<AssociableEntityKey, Entity> localAssociableEntityMap = new HashMap<>();
List<EntityCandidates> entityCandidateList = new ArrayList<>();
for (EntityMember ingestionIdEntity : ingestionIdEntities) {
Entity innerEntity = ingestionIdEntity.getEntity();
entityCandidateList.add(getEntityCandidates(entityNameCleanerProvider, entityService, ingestionIdEntity, innerEntity));
}
List<Mono<Entity>> completableFutureList = entityCandidateList.stream().filter(entityCandidates -> entityCandidates.getEntity().isLocal()).map(entityCandidates -> Mono.fromSupplier(_getAndCreateEntityDisambiguate(config, entityNameCleanerProvider, entityService, entityGraphService, entityCandidateList, entityCandidates, entityCandidates.getEntity())).subscribeOn(Schedulers.boundedElastic())).collect(Collectors.toList());
Mono<List<Entity>> zip = Mono.zip(completableFutureList, a -> {
List<Entity> entities = new ArrayList<>();
for (Object o : a) {
entities.add((Entity) o);
}
return entities;
}).defaultIfEmpty(List.of());
for (Entity currentEntityRequest : zip.block()) {
localAssociableEntityMap.put(AssociableEntityKey.of(currentEntityRequest.getCacheId(), currentEntityRequest.getIngestionId()), currentEntityRequest);
entityMap.put(EntityKey.of(currentEntityRequest.getTenantId(), currentEntityRequest.getName(), currentEntityRequest.getType(), currentEntityRequest.getCacheId(), currentEntityRequest.getIngestionId()), currentEntityRequest);
}
associableEntityMap.setAll(localAssociableEntityMap);
}
entityIMap.setAll(entityMap);
}
use of io.openk9.entity.manager.service.graph.EntityGraphService in project openk9 by smclab.
the class CreateRelationRunnable method run_.
@Override
public void run_() {
IMap<EntityKey, Entity> entityIMap = MapUtil.getEntityMap(_hazelcastInstance);
Set<EntityKey> entityKeys = entityIMap.localKeySet(Predicates.and(Predicates.notEqual("id", null), Predicates.notEqual("graphId", null)));
_log.info("entityKeys: " + entityKeys.size());
Map<EntityKey, Entity> entityIMapAll = entityIMap.getAll(entityKeys);
_log.info("entityIMapAll: " + entityIMapAll.size());
IMap<EntityRelationKey, EntityRelation> entityRelationMap = MapUtil.getEntityRelationMap(_hazelcastInstance);
Map<String, String> collect = entityIMapAll.values().stream().collect(Collectors.toMap(Entity::getCacheId, Entity::getId));
String[] cacheIds = entityKeys.stream().map(EntityKey::getCacheId).distinct().toArray(String[]::new);
Map<EntityRelationKey, EntityRelation> entries = entityRelationMap.getAll(entityRelationMap.keySet(Predicates.in("__key.entityId", cacheIds)));
_log.info("entityRelations: " + entries.size());
EntityGraphService entityGraphService = CDI.current().select(EntityGraphService.class).get();
List<EntityRelationKey> entityRelationKeysToDelete = new ArrayList<>();
for (Map.Entry<EntityRelationKey, EntityRelation> entry : entries.entrySet()) {
EntityRelationKey key = entry.getKey();
EntityRelation value = entry.getValue();
String from = collect.get(value.getEntityCacheId());
String to = collect.get(value.getTo());
if (from != null && to != null) {
try {
entityGraphService.createRelationship(from, to, value.getName());
entityRelationKeysToDelete.add(key);
} catch (Exception e) {
_log.error(e.getMessage(), e);
}
}
}
try {
Pipelining pipelining = new Pipelining<>(10);
for (EntityRelationKey entityRelationKey : entityRelationKeysToDelete) {
pipelining.add(entityRelationMap.removeAsync(entityRelationKey));
}
pipelining.results();
} catch (Exception e) {
_log.error(e.getMessage(), e);
}
}
Aggregations