use of io.openk9.datasource.model.Datasource in project openk9 by smclab.
the class GetOrAddEntitiesConsumer method apply.
public Mono<ObjectNode> apply(ObjectNode objectNode) {
return Mono.defer(() -> {
ObjectNode datasourceContextJson = objectNode.get("datasourceContext").toObjectNode();
long datasourceId = datasourceContextJson.get("datasource").get("datasourceId").asLong();
long tenantId = datasourceContextJson.get("tenant").get("tenantId").asLong();
JsonNode entities = objectNode.remove("entities");
Mono<ArrayNode> entitiesField;
if (entities.size() == 0) {
entitiesField = Mono.just(_jsonFactory.createArrayNode());
} else {
ObjectNode responseJson = _jsonFactory.createObjectNode();
responseJson.put("entities", entities);
responseJson.put("tenantId", tenantId);
responseJson.put("datasourceId", datasourceId);
Request request = _jsonFactory.fromJson(responseJson.toString(), Request.class);
List<RequestContext> requestContextList = request.getEntities().stream().map(entityRequest -> RequestContext.builder().current(entityRequest).tenantId(request.getTenantId()).datasourceId(request.getDatasourceId()).rest(request.getEntities().stream().filter(er -> er != entityRequest).collect(Collectors.toList())).build()).collect(Collectors.toList());
Mono<List<EntityContext>> disambiguateListMono = GetOrAddEntities.stopWatch("disambiguate-all-entities", Flux.fromIterable(requestContextList).flatMap(requestContext -> GetOrAddEntities.stopWatch("disambiguate-" + requestContext.getCurrent().getName(), Mono.<EntityContext>create(fluxSink -> _startDisambiguation.disambiguate(requestContext, fluxSink)))).collectList());
Mono<ResponseList> writeRelations = disambiguateListMono.flatMap(entityContexts -> GetOrAddEntities.stopWatch("write-relations", writeRelations(entityContexts)));
Mono<ResponseList> responseListWrapper = _transactional ? _graphClient.makeTransactional(writeRelations) : writeRelations;
entitiesField = responseListWrapper.map(responseListDTO -> {
List<Response> responseList = responseListDTO.getResponse();
ArrayNode entitiesArrayNode = entities.toArrayNode();
ArrayNode arrayNode = _jsonFactory.createArrayNode();
for (JsonNode node : entitiesArrayNode) {
Optional<Response> responseOptional = responseList.stream().filter(response -> node.get("tmpId").asLong() == response.getTmpId()).findFirst();
if (responseOptional.isPresent()) {
Entity entity = responseOptional.get().getEntity();
ObjectNode result = _jsonFactory.createObjectNode();
result.put("entityType", entity.getType());
result.put("id", entity.getId());
result.put("context", node.get("context"));
arrayNode.add(result);
}
}
return arrayNode;
});
}
return entitiesField.map(entitiesArray -> {
ObjectNode payload = objectNode.get("payload").toObjectNode();
payload.set("entities", entitiesArray);
return objectNode;
}).timeout(Duration.ofSeconds(_timeout), Mono.error(new TimeoutException("timeout on entities count: " + entities.size() + " (Did not observe any item or terminal signal within " + Duration.ofSeconds(_timeout).toMillis() + "ms)")));
});
}
use of io.openk9.datasource.model.Datasource in project openk9 by smclab.
the class ReindexDatasourceConsumer method activate.
@Activate
void activate() {
RestHighLevelClient restHighLevelClient = _restHighLevelClientProvider.get();
_disposable = _datasourceEventConsumer.datasourceUpdateEvents().flatMap(datasource -> _pluginDriverManagerClient.getPluginDriver(datasource.getDriverServiceName()).onErrorResume(throwable -> {
if (_log.isErrorEnabled()) {
_log.error(throwable.getMessage());
}
return Mono.empty();
}).map(pluginDriverDTO -> Tuples.of(datasource, datasource.getTenantId() + "-" + pluginDriverDTO.getName() + "-data"))).filterWhen(t2 -> Mono.create(sink -> restHighLevelClient.indices().existsAsync(new GetIndexRequest(t2.getT2()), RequestOptions.DEFAULT, new ReactorActionListener<>(sink)))).flatMap(t2 -> Mono.<GetSettingsResponse>create(sink -> restHighLevelClient.indices().getSettingsAsync(new GetSettingsRequest().indices(t2.getT2()).names("index.creation_date"), RequestOptions.DEFAULT, new ReactorActionListener<>(sink))).map(response -> Tuples.of(t2.getT1(), t2.getT2(), Instant.ofEpochMilli(Long.parseLong(response.getSetting(t2.getT2(), "index.creation_date")))))).log().filter(t3 -> t3.getT1().getLastIngestionDate().isBefore(t3.getT3())).flatMap(t3 -> Mono.<AcknowledgedResponse>create(sink -> restHighLevelClient.indices().deleteAsync(new DeleteIndexRequest(t3.getT2()), RequestOptions.DEFAULT, new ReactorActionListener<>(sink)))).onErrorContinue((throwable, ignore) -> {
if (_log.isErrorEnabled()) {
_log.error(throwable.getMessage());
}
}).subscribe();
}
use of io.openk9.datasource.model.Datasource in project openk9 by smclab.
the class SuggestionsHTTPHandler method _getSuggestionsResponse.
private SuggestionsResponse _getSuggestionsResponse(List<Datasource> datasourceList, PluginDriverDTOList pluginDriverDTOList, SearchRequest searchRequest, SearchResponse searchResponse, List<SuggestionCategoryField> fields, Map<String, String[]> entityMap) {
Aggregations aggregations = searchResponse.getAggregations();
if (aggregations == null) {
return SuggestionsResponse.of(List.of(), null);
}
Map<String, Aggregation> aggregationMap = aggregations.asMap();
if (!aggregationMap.containsKey("composite")) {
return SuggestionsResponse.of(List.of(), null);
}
CompositeAggregation compositeAggregation = (CompositeAggregation) aggregationMap.get("composite");
Map<String, Long> fieldNameCategoryIdMap = fields.stream().collect(Collectors.toMap(SuggestionCategoryField::getFieldName, SuggestionCategoryField::getCategoryId, (a1, a2) -> a2));
Long datasourceIdCategoryId = fieldNameCategoryIdMap.getOrDefault("datasourceId", 1L);
Long entityIdCategoryId = fieldNameCategoryIdMap.getOrDefault("entities.id", 2L);
Long entitiesContextCategoryId = fieldNameCategoryIdMap.getOrDefault("entities.context", entityIdCategoryId);
Long documentTypesCategoryId = fieldNameCategoryIdMap.getOrDefault("documentTypes", 4L);
List<? extends CompositeAggregation.Bucket> buckets = compositeAggregation.getBuckets();
LinkedList<Suggestions> suggestions = new LinkedList<>();
String suggestKeyword = searchRequest.getSuggestKeyword();
BiConsumer<String, Suggestions> addSuggestions;
if (suggestKeyword != null) {
addSuggestions = (key, sugg) -> {
if (!suggestions.contains(sugg)) {
if (key.contains(suggestKeyword)) {
suggestions.addFirst(sugg);
}
}
};
} else {
addSuggestions = (key, sugg) -> {
if (!suggestions.contains(sugg)) {
suggestions.add(sugg);
}
};
}
for (CompositeAggregation.Bucket bucket : buckets) {
Map<String, Object> keys = new HashMap<>(bucket.getKey());
for (Map.Entry<String, Object> entry : keys.entrySet()) {
String key = entry.getKey();
String value = (String) entry.getValue();
if (value == null) {
continue;
}
switch(key) {
case "datasourceId":
long datasourceIdL = Long.parseLong(value);
_datasource(datasourceList, pluginDriverDTOList, datasourceIdCategoryId, addSuggestions, datasourceIdL);
break;
case "entities.context":
break;
case "entities.id":
String[] typeName = entityMap.get(value);
if (typeName != null) {
String type = typeName[0];
String name = typeName[1];
String entitiesContext = (String) keys.get("entities.context");
if (entitiesContext != null) {
addSuggestions.accept(name, Suggestions.entity(value, entitiesContextCategoryId, type, name, entitiesContext));
} else {
addSuggestions.accept(name, Suggestions.entity(value, entityIdCategoryId, type, name));
}
}
break;
case "documentTypes":
addSuggestions.accept(value, Suggestions.docType(value, documentTypesCategoryId));
break;
default:
Long textCategoryId = fieldNameCategoryIdMap.getOrDefault(key, 5L);
addSuggestions.accept(value, Suggestions.text(value, textCategoryId, key));
}
}
}
Map<String, Object> map = compositeAggregation.afterKey();
String afterKey = null;
int[] range = searchRequest.getRange();
if (map != null) {
afterKey = _jsonFactory.toJson(map);
afterKey = Base64.getEncoder().encodeToString(afterKey.getBytes(StandardCharsets.UTF_8));
}
return SuggestionsResponse.of(suggestions, afterKey);
}
use of io.openk9.datasource.model.Datasource in project openk9 by smclab.
the class SupportedDatasourcesEndpoint method _mapToResponse.
private SupportedDatasourcesResponse _mapToResponse(Tuple2<Datasource, PluginDriverDTO> t2) {
Datasource t1 = t2.getT1();
PluginDriverDTO pluginDriver = t2.getT2();
return SupportedDatasourcesResponse.of(t1.getName(), t1.getActive(), pluginDriver.getDocumentTypes().stream().map(documentTypeDTO -> SupportedDatasourcesDocumentType.of(documentTypeDTO.getName(), documentTypeDTO.getIcon())).collect(Collectors.toList()), SupportedDatasourcesDocumentType.of(pluginDriver.getDefaultDocumentType().getName(), pluginDriver.getDefaultDocumentType().getIcon()));
}
use of io.openk9.datasource.model.Datasource in project openk9 by smclab.
the class DocumentTypesEndpoint method _mapToResponse.
private Object _mapToResponse(List<Tuple2<Datasource, PluginDriverDTO>> list) {
Map<String, Collection<String>> response = new HashMap<>();
for (Tuple2<Datasource, PluginDriverDTO> t2 : list) {
PluginDriverDTO pluginDriver = t2.getT2();
for (DocumentTypeDTO documentType : pluginDriver.getDocumentTypes()) {
String name = documentType.getName();
List<SearchKeywordDTO> searchKeywords = documentType.getSearchKeywords();
List<String> keywords = searchKeywords.stream().map(SearchKeywordDTO::getKeyword).collect(Collectors.toList());
Collection<String> prevKeywords = response.get(name);
Set<String> combine = new HashSet<>(keywords);
if (prevKeywords != null) {
combine.addAll(prevKeywords);
}
response.put(name, combine);
}
}
return response;
}
Aggregations