Examples with Datasource io.openk9.model.Datasource used on opensource projects

Example 1 with Datasource

use of io.openk9.model.Datasource in project openk9 by smclab.

the class GetOrAddEntitiesConsumer method apply.

public Mono<ObjectNode> apply(ObjectNode objectNode) {
    return Mono.defer(() -> {
        ObjectNode datasourceContextJson = objectNode.get("datasourceContext").toObjectNode();
        long datasourceId = datasourceContextJson.get("datasource").get("datasourceId").asLong();
        long tenantId = datasourceContextJson.get("tenant").get("tenantId").asLong();
        JsonNode entities = objectNode.remove("entities");
        Mono<ArrayNode> entitiesField;
        if (entities.size() == 0) {
            entitiesField = Mono.just(_jsonFactory.createArrayNode());
        } else {
            ObjectNode responseJson = _jsonFactory.createObjectNode();
            responseJson.put("entities", entities);
            responseJson.put("tenantId", tenantId);
            responseJson.put("datasourceId", datasourceId);
            Request request = _jsonFactory.fromJson(responseJson.toString(), Request.class);
            List<RequestContext> requestContextList = request.getEntities().stream().map(entityRequest -> RequestContext.builder().current(entityRequest).tenantId(request.getTenantId()).datasourceId(request.getDatasourceId()).rest(request.getEntities().stream().filter(er -> er != entityRequest).collect(Collectors.toList())).build()).collect(Collectors.toList());
            Mono<List<EntityContext>> disambiguateListMono = GetOrAddEntities.stopWatch("disambiguate-all-entities", Flux.fromIterable(requestContextList).flatMap(requestContext -> GetOrAddEntities.stopWatch("disambiguate-" + requestContext.getCurrent().getName(), Mono.<EntityContext>create(fluxSink -> _startDisambiguation.disambiguate(requestContext, fluxSink)))).collectList());
            Mono<ResponseList> writeRelations = disambiguateListMono.flatMap(entityContexts -> GetOrAddEntities.stopWatch("write-relations", writeRelations(entityContexts)));
            Mono<ResponseList> responseListWrapper = _transactional ? _graphClient.makeTransactional(writeRelations) : writeRelations;
            entitiesField = responseListWrapper.map(responseListDTO -> {
                List<Response> responseList = responseListDTO.getResponse();
                ArrayNode entitiesArrayNode = entities.toArrayNode();
                ArrayNode arrayNode = _jsonFactory.createArrayNode();
                for (JsonNode node : entitiesArrayNode) {
                    Optional<Response> responseOptional = responseList.stream().filter(response -> node.get("tmpId").asLong() == response.getTmpId()).findFirst();
                    if (responseOptional.isPresent()) {
                        Entity entity = responseOptional.get().getEntity();
                        ObjectNode result = _jsonFactory.createObjectNode();
                        result.put("entityType", entity.getType());
                        result.put("id", entity.getId());
                        result.put("context", node.get("context"));
                        arrayNode.add(result);
                    }
                }
                return arrayNode;
            });
        }
        return entitiesField.map(entitiesArray -> {
            ObjectNode payload = objectNode.get("payload").toObjectNode();
            payload.set("entities", entitiesArray);
            return objectNode;
        }).timeout(Duration.ofSeconds(_timeout), Mono.error(new TimeoutException("timeout on entities count: " + entities.size() + " (Did not observe any item or terminal signal within " + Duration.ofSeconds(_timeout).toMillis() + "ms)")));
    });
}

Example 2 with Datasource

use of io.openk9.model.Datasource in project openk9 by smclab.

the class ReindexDatasourceConsumer method activate.

@Activate
void activate() {
    RestHighLevelClient restHighLevelClient = _restHighLevelClientProvider.get();
    _disposable = _datasourceEventConsumer.datasourceUpdateEvents().flatMap(datasource -> _pluginDriverManagerClient.getPluginDriver(datasource.getDriverServiceName()).onErrorResume(throwable -> {
        if (_log.isErrorEnabled()) {
            _log.error(throwable.getMessage());
        }
        return Mono.empty();
    }).map(pluginDriverDTO -> Tuples.of(datasource, datasource.getTenantId() + "-" + pluginDriverDTO.getName() + "-data"))).filterWhen(t2 -> Mono.create(sink -> restHighLevelClient.indices().existsAsync(new GetIndexRequest(t2.getT2()), RequestOptions.DEFAULT, new ReactorActionListener<>(sink)))).flatMap(t2 -> Mono.<GetSettingsResponse>create(sink -> restHighLevelClient.indices().getSettingsAsync(new GetSettingsRequest().indices(t2.getT2()).names("index.creation_date"), RequestOptions.DEFAULT, new ReactorActionListener<>(sink))).map(response -> Tuples.of(t2.getT1(), t2.getT2(), Instant.ofEpochMilli(Long.parseLong(response.getSetting(t2.getT2(), "index.creation_date")))))).log().filter(t3 -> t3.getT1().getLastIngestionDate().isBefore(t3.getT3())).flatMap(t3 -> Mono.<AcknowledgedResponse>create(sink -> restHighLevelClient.indices().deleteAsync(new DeleteIndexRequest(t3.getT2()), RequestOptions.DEFAULT, new ReactorActionListener<>(sink)))).onErrorContinue((throwable, ignore) -> {
        if (_log.isErrorEnabled()) {
            _log.error(throwable.getMessage());
        }
    }).subscribe();
}

Example 3 with Datasource

use of io.openk9.model.Datasource in project openk9 by smclab.

the class SuggestionsHTTPHandler method _getSuggestionsResponse.

private SuggestionsResponse _getSuggestionsResponse(List<Datasource> datasourceList, PluginDriverDTOList pluginDriverDTOList, SearchRequest searchRequest, SearchResponse searchResponse, List<SuggestionCategoryField> fields, Map<String, String[]> entityMap) {
    Aggregations aggregations = searchResponse.getAggregations();
    if (aggregations == null) {
        return SuggestionsResponse.of(List.of(), null);
    }
    Map<String, Aggregation> aggregationMap = aggregations.asMap();
    if (!aggregationMap.containsKey("composite")) {
        return SuggestionsResponse.of(List.of(), null);
    }
    CompositeAggregation compositeAggregation = (CompositeAggregation) aggregationMap.get("composite");
    Map<String, Long> fieldNameCategoryIdMap = fields.stream().collect(Collectors.toMap(SuggestionCategoryField::getFieldName, SuggestionCategoryField::getCategoryId, (a1, a2) -> a2));
    Long datasourceIdCategoryId = fieldNameCategoryIdMap.getOrDefault("datasourceId", 1L);
    Long entityIdCategoryId = fieldNameCategoryIdMap.getOrDefault("entities.id", 2L);
    Long entitiesContextCategoryId = fieldNameCategoryIdMap.getOrDefault("entities.context", entityIdCategoryId);
    Long documentTypesCategoryId = fieldNameCategoryIdMap.getOrDefault("documentTypes", 4L);
    List<? extends CompositeAggregation.Bucket> buckets = compositeAggregation.getBuckets();
    LinkedList<Suggestions> suggestions = new LinkedList<>();
    String suggestKeyword = searchRequest.getSuggestKeyword();
    BiConsumer<String, Suggestions> addSuggestions;
    if (suggestKeyword != null) {
        addSuggestions = (key, sugg) -> {
            if (!suggestions.contains(sugg)) {
                if (key.contains(suggestKeyword)) {
                    suggestions.addFirst(sugg);
                }
            }
        };
    } else {
        addSuggestions = (key, sugg) -> {
            if (!suggestions.contains(sugg)) {
                suggestions.add(sugg);
            }
        };
    }
    for (CompositeAggregation.Bucket bucket : buckets) {
        Map<String, Object> keys = new HashMap<>(bucket.getKey());
        for (Map.Entry<String, Object> entry : keys.entrySet()) {
            String key = entry.getKey();
            String value = (String) entry.getValue();
            if (value == null) {
                continue;
            }
            switch(key) {
                case "datasourceId":
                    long datasourceIdL = Long.parseLong(value);
                    _datasource(datasourceList, pluginDriverDTOList, datasourceIdCategoryId, addSuggestions, datasourceIdL);
                    break;
                case "entities.context":
                    break;
                case "entities.id":
                    String[] typeName = entityMap.get(value);
                    if (typeName != null) {
                        String type = typeName[0];
                        String name = typeName[1];
                        String entitiesContext = (String) keys.get("entities.context");
                        if (entitiesContext != null) {
                            addSuggestions.accept(name, Suggestions.entity(value, entitiesContextCategoryId, type, name, entitiesContext));
                        } else {
                            addSuggestions.accept(name, Suggestions.entity(value, entityIdCategoryId, type, name));
                        }
                    }
                    break;
                case "documentTypes":
                    addSuggestions.accept(value, Suggestions.docType(value, documentTypesCategoryId));
                    break;
                default:
                    Long textCategoryId = fieldNameCategoryIdMap.getOrDefault(key, 5L);
                    addSuggestions.accept(value, Suggestions.text(value, textCategoryId, key));
            }
        }
    }
    Map<String, Object> map = compositeAggregation.afterKey();
    String afterKey = null;
    int[] range = searchRequest.getRange();
    if (map != null) {
        afterKey = _jsonFactory.toJson(map);
        afterKey = Base64.getEncoder().encodeToString(afterKey.getBytes(StandardCharsets.UTF_8));
    }
    return SuggestionsResponse.of(suggestions, afterKey);
}

Example 4 with Datasource

use of io.openk9.model.Datasource in project openk9 by smclab.

the class SupportedDatasourcesEndpoint method _mapToResponse.

private SupportedDatasourcesResponse _mapToResponse(Tuple2<Datasource, PluginDriverDTO> t2) {
    Datasource t1 = t2.getT1();
    PluginDriverDTO pluginDriver = t2.getT2();
    return SupportedDatasourcesResponse.of(t1.getName(), t1.getActive(), pluginDriver.getDocumentTypes().stream().map(documentTypeDTO -> SupportedDatasourcesDocumentType.of(documentTypeDTO.getName(), documentTypeDTO.getIcon())).collect(Collectors.toList()), SupportedDatasourcesDocumentType.of(pluginDriver.getDefaultDocumentType().getName(), pluginDriver.getDefaultDocumentType().getIcon()));
}

Example 5 with Datasource

use of io.openk9.model.Datasource in project openk9 by smclab.

the class DocumentTypesEndpoint method _mapToResponse.

private Object _mapToResponse(List<Tuple2<Datasource, PluginDriverDTO>> list) {
    Map<String, Collection<String>> response = new HashMap<>();
    for (Tuple2<Datasource, PluginDriverDTO> t2 : list) {
        PluginDriverDTO pluginDriver = t2.getT2();
        for (DocumentTypeDTO documentType : pluginDriver.getDocumentTypes()) {
            String name = documentType.getName();
            List<SearchKeywordDTO> searchKeywords = documentType.getSearchKeywords();
            List<String> keywords = searchKeywords.stream().map(SearchKeywordDTO::getKeyword).collect(Collectors.toList());
            Collection<String> prevKeywords = response.get(name);
            Set<String> combine = new HashSet<>(keywords);
            if (prevKeywords != null) {
                combine.addAll(prevKeywords);
            }
            response.put(name, combine);
        }
    }
    return response;
}