use of io.openk9.entity.manager.cache.model.AssociableEntityKey in project openk9 by smclab.
the class AssociateEntitiesRunnable method run_.
@Override
public void run_() {
_log.info("start AssociateEntitiesRunnable");
IMap<AssociableEntityKey, Entity> associableEntityMap = MapUtil.getAssociableEntityMap(_hazelcastInstance);
Set<AssociableEntityKey> associableEntityKeys = associableEntityMap.localKeySet();
Map<AssociableEntityKey, Entity> localEntityMap = associableEntityMap.getAll(associableEntityKeys);
_log.info("ingestionKeys: " + localEntityMap.size());
Map<String, List<Entity>> groupingByIngestionId = localEntityMap.entrySet().stream().collect(Collectors.groupingBy(e -> e.getKey().getIngestionId(), Collectors.mapping(Map.Entry::getValue, Collectors.toList())));
List<AssociableEntityKey> entitiesToRemove = new ArrayList<>();
List<String> ingestionIds = new ArrayList<>();
for (Map.Entry<String, List<Entity>> entry : groupingByIngestionId.entrySet()) {
String ingestionId = entry.getKey();
List<Entity> v = entry.getValue();
if (v.isEmpty()) {
continue;
}
DataService dataService = CDI.current().select(DataService.class).get();
Long tenantId = v.stream().map(Entity::getTenantId).findFirst().get();
try {
boolean associated = dataService.associateEntities(tenantId, ingestionId, v.stream().map(IngestionEntity::fromEntity).collect(Collectors.toList()));
if (associated) {
for (Entity entity : v) {
entitiesToRemove.add(AssociableEntityKey.of(entity.getCacheId(), entity.getIngestionId()));
ingestionIds.add(ingestionId);
}
}
} catch (Exception ioe) {
_log.error(ioe.getMessage());
}
}
_log.info("entities associated: " + entitiesToRemove.size() + " ingestionIds: " + ingestionIds);
try {
Pipelining pipelining = new Pipelining<>(10);
for (AssociableEntityKey associateEntityKey : entitiesToRemove) {
pipelining.add(associableEntityMap.removeAsync(associateEntityKey));
}
pipelining.results();
} catch (Exception e) {
_log.error(e.getMessage(), e);
}
}
use of io.openk9.entity.manager.cache.model.AssociableEntityKey in project openk9 by smclab.
the class CreateEntitiesRunnable method run_.
@Override
public void run_() {
_log.info("start CreateEntitiesRunnable");
IMap<EntityKey, Entity> entityIMap = MapUtil.getEntityMap(_hazelcastInstance);
IMap<AssociableEntityKey, Entity> associableEntityMap = MapUtil.getAssociableEntityMap(_hazelcastInstance);
Set<EntityKey> entityKeys = entityIMap.localKeySet(Predicates.and(Predicates.equal("id", null), Predicates.equal("graphId", null)));
EntityGraphConfig config = CDI.current().select(EntityGraphConfig.class).get();
EntityNameCleanerProvider entityNameCleanerProvider = CDI.current().select(EntityNameCleanerProvider.class).get();
EntityService entityService = CDI.current().select(EntityService.class).get();
EntityGraphService entityGraphService = CDI.current().select(EntityGraphService.class).get();
Map<EntityKey, Entity> localEntityMap = entityIMap.getAll(entityKeys);
Collection<Entity> localEntityValues = localEntityMap.values();
Set<EntityKey> localEntityKeys = localEntityMap.keySet();
List<Member> collect = _hazelcastInstance.getCluster().getMembers().stream().filter(member -> !member.localMember()).collect(Collectors.toList());
String[] ingestionIds = localEntityKeys.stream().map(EntityKey::getIngestionId).distinct().toArray(String[]::new);
IExecutorService entityExecutor = _hazelcastInstance.getExecutorService("entityExecutor");
Map<Member, Future<Map<EntityKey, Entity>>> memberFutureMap = entityExecutor.submitToMembers(new GetEntitiesCallable(ingestionIds), collect);
Map<EntityKey, Entity> otherEntityKeyEntityMap = memberFutureMap.values().stream().map(FutureUtil::makeCompletableFuture).map(CompletableFuture::join).reduce((a, b) -> {
Map<EntityKey, Entity> map = new HashMap<>();
map.putAll(a);
map.putAll(b);
return map;
}).orElseGet(Map::of);
Stream<EntityMember> otherEntityMemberStream = otherEntityKeyEntityMap.values().stream().map(entity -> EntityMember.of(entity, false));
Stream<EntityMember> localEntityMemberStream = localEntityValues.stream().map(entity -> EntityMember.of(entity, true));
Map<String, List<EntityMember>> entitiesGroupingByIngestionId = Stream.concat(localEntityMemberStream, otherEntityMemberStream).collect(Collectors.groupingBy(entityMember -> entityMember.getEntity().getIngestionId()));
Collection<List<EntityMember>> values = entitiesGroupingByIngestionId.values();
Map<EntityKey, Entity> entityMap = new HashMap<>();
for (List<EntityMember> ingestionIdEntities : values) {
Map<AssociableEntityKey, Entity> localAssociableEntityMap = new HashMap<>();
List<EntityCandidates> entityCandidateList = new ArrayList<>();
for (EntityMember ingestionIdEntity : ingestionIdEntities) {
Entity innerEntity = ingestionIdEntity.getEntity();
entityCandidateList.add(getEntityCandidates(entityNameCleanerProvider, entityService, ingestionIdEntity, innerEntity));
}
List<Mono<Entity>> completableFutureList = entityCandidateList.stream().filter(entityCandidates -> entityCandidates.getEntity().isLocal()).map(entityCandidates -> Mono.fromSupplier(_getAndCreateEntityDisambiguate(config, entityNameCleanerProvider, entityService, entityGraphService, entityCandidateList, entityCandidates, entityCandidates.getEntity())).subscribeOn(Schedulers.boundedElastic())).collect(Collectors.toList());
Mono<List<Entity>> zip = Mono.zip(completableFutureList, a -> {
List<Entity> entities = new ArrayList<>();
for (Object o : a) {
entities.add((Entity) o);
}
return entities;
}).defaultIfEmpty(List.of());
for (Entity currentEntityRequest : zip.block()) {
localAssociableEntityMap.put(AssociableEntityKey.of(currentEntityRequest.getCacheId(), currentEntityRequest.getIngestionId()), currentEntityRequest);
entityMap.put(EntityKey.of(currentEntityRequest.getTenantId(), currentEntityRequest.getName(), currentEntityRequest.getType(), currentEntityRequest.getCacheId(), currentEntityRequest.getIngestionId()), currentEntityRequest);
}
associableEntityMap.setAll(localAssociableEntityMap);
}
entityIMap.setAll(entityMap);
}
Aggregations