use of io.openk9.model.Datasource in project openk9 by smclab.
the class IndexWriterEndpoins method _deleteDataDocuments.
private Publisher<Void> _deleteDataDocuments(HttpServerRequest httpServerRequest, HttpServerResponse httpServerResponse) {
RestHighLevelClient restHighLevelClient = _restHighLevelClientProvider.get();
Mono<byte[]> body = ReactorNettyUtils.aggregateBodyAsByteArray(httpServerRequest);
Mono<String> responseMono = body.map(bytes -> _jsonFactory.fromJson(bytes, DeleteEntitiesRequest.class)).flatMap(deleteEntitiesRequest -> {
if (!deleteEntitiesRequest.getContentIds().isEmpty()) {
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
for (String contentId : deleteEntitiesRequest.getContentIds()) {
boolQuery.mustNot(matchQuery("contentId", contentId));
}
Mono<Datasource> datasourceMono = _datasourceClient.findDatasource(deleteEntitiesRequest.getDatasourceId());
return datasourceMono.flatMap(datasource -> _pluginDriverManagerClient.getPluginDriver(datasource.getDriverServiceName()).map(pluginDriverDTO -> datasource.getTenantId() + "-" + pluginDriverDTO.getName() + "-data").map(DeleteByQueryRequest::new).map(deleteByQueryRequest -> deleteByQueryRequest.setQuery(boolQuery)).flatMap(deleteByQueryRequest -> Mono.<BulkByScrollResponse>create(sink -> restHighLevelClient.deleteByQueryAsync(deleteByQueryRequest, RequestOptions.DEFAULT, new ReactorActionListener<>(sink)))).map(Object::toString).doOnNext(_log::info).flatMap(response -> Mono.fromRunnable(() -> _sendCleanOrphanEntitiesRequest(datasource.getTenantId())).thenReturn(response)));
}
return Mono.empty();
});
return _httpResponseWriter.write(httpServerResponse, responseMono);
}
use of io.openk9.model.Datasource in project openk9 by smclab.
the class DriverManagerActivator method activate.
@Activate
public void activate(Config config) throws SchedulerError {
Disposable disposable1 = _datasourceRepository.findAll(true).concatMap(this::_schedule).subscribe();
Disposable disposable2 = _entityEventBus.stream().filter(e -> e.getEntityClass() == Datasource.class).concatMap(entityEvent -> {
Datasource datasource = (Datasource) entityEvent.getValue();
if (entityEvent instanceof EntityEvent.UpdateEvent || entityEvent instanceof EntityEvent.InsertEvent) {
return _schedule(datasource);
} else {
return Mono.fromRunnable(() -> _scheduler.unschedule(_PREFIX + datasource.getName()));
}
}).subscribe();
_autoClosableSafe = AutoCloseables.mergeAutoCloseableToSafe(disposable1::dispose, disposable2::dispose);
}
use of io.openk9.model.Datasource in project openk9 by smclab.
the class BaseSearchHTTPHandler method _toQuerySearchRequest.
private Mono<SearchResponse> _toQuerySearchRequest(Tenant tenant, List<Datasource> datasources, PluginDriverDTOList pdDTOList, SearchRequest searchRequest, HttpServerRequest httpRequest) {
return Mono.defer(() -> {
List<PluginDriverDTO> pluginDriverDTOList = pdDTOList.getPluginDriverDTOList();
Map<String, List<SearchToken>> tokenTypeGroup = searchRequest.getSearchQuery().stream().collect(Collectors.groupingBy(SearchToken::getTokenType));
List<SearchToken> datasource = tokenTypeGroup.get("DATASOURCE");
Stream<PluginDriverDTO> documentTypeStream = pluginDriverDTOList.stream();
if (datasource != null) {
List<String> datasourceValues = datasource.stream().map(SearchToken::getValues).flatMap(Arrays::stream).distinct().collect(Collectors.toList());
documentTypeStream = documentTypeStream.filter(entry -> datasourceValues.contains(entry.getName()));
}
List<PluginDriverDTO> documentTypeList = documentTypeStream.collect(Collectors.toList());
QueryParser queryParser = _queryParsers.stream().reduce(QueryParser.NOTHING, QueryParser::andThen);
return queryParser.apply(createQueryParserContext(tenant, datasources, httpRequest, tokenTypeGroup, documentTypeList)).flatMap(boolQueryBuilderConsumer -> _search.flatMapSearch(factory -> {
long tenantId = tenant.getTenantId();
if (documentTypeList.isEmpty()) {
return Mono.just(SearchUtil.EMPTY_SEARCH_REQUEST);
}
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
boolQueryBuilderConsumer.accept(boolQuery);
org.elasticsearch.action.search.SearchRequest elasticSearchQuery;
if (datasource != null) {
String[] indexNames = documentTypeList.stream().map(PluginDriverDTO::getName).distinct().toArray(String[]::new);
elasticSearchQuery = factory.createSearchRequestData(tenantId, indexNames);
} else {
elasticSearchQuery = factory.createSearchRequestData(tenantId, "*");
}
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.fetchSource(includeFields(), excludeFields());
searchSourceBuilder.query(boolQuery);
searchSourceBuilder.trackTotalHits(true);
return customizeSearchSourceBuilderMono(tenant, datasources, searchRequest, documentTypeList, searchSourceBuilder, elasticSearchQuery);
}));
});
}
use of io.openk9.model.Datasource in project openk9 by smclab.
the class BaseSearchHTTPHandler method customizeSearchSourceBuilder.
protected void customizeSearchSourceBuilder(Tenant tenant, List<Datasource> datasources, SearchRequest searchRequest, List<PluginDriverDTO> documentTypeList, SearchSourceBuilder searchSourceBuilder, org.elasticsearch.action.search.SearchRequest elasticSearchQuery) {
int[] range = searchRequest.getRange();
if (range != null) {
searchSourceBuilder.from(range[0]);
searchSourceBuilder.size(range[1]);
}
HighlightBuilder highlightBuilder = new HighlightBuilder();
documentTypeList.stream().map(PluginDriverDTO::getDocumentTypes).flatMap(Collection::stream).map(DocumentTypeDTO::getSearchKeywords).flatMap(Collection::stream).filter(SearchKeywordDTO::isText).map(SearchKeywordDTO::getKeyword).distinct().forEach(highlightBuilder::field);
highlightBuilder.forceSource(true);
highlightBuilder.tagsSchema("default");
searchSourceBuilder.highlighter(highlightBuilder);
}
use of io.openk9.model.Datasource in project openk9 by smclab.
the class ResourcesHttpHandler method _sendResource.
private Mono<Void> _sendResource(Tenant tenant, Datasource datasource, PluginDriverDTO pluginDriverDTO, long datasourceId, String documentId, String resourceId, HttpServerRequest httpRequest, HttpServerResponse httpResponse) {
return _queryParser.apply(QueryParser.Context.of(tenant, null, null, null, httpRequest, QueryParser.QueryCondition.DEFAULT)).flatMap(consumer -> _search.search(factory -> {
SearchRequest searchRequest = factory.createSearchRequestData(tenant.getTenantId(), pluginDriverDTO.getName());
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.idsQuery().addIds(documentId));
boolQueryBuilder.must(QueryBuilders.nestedQuery(_RESOURCES_BINARIES, QueryBuilders.matchQuery(_RESOURCES_BINARIES_ID, resourceId), ScoreMode.Max));
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
consumer.accept(boolQueryBuilder);
searchSourceBuilder.query(boolQueryBuilder);
searchSourceBuilder.fetchSource(new String[] { _RESOURCES_BINARIES_DATA, _RESOURCES_BINARIES_CONTENT_TYPE }, null);
return searchRequest.source(searchSourceBuilder);
})).flatMap(response -> {
SearchHits searchHits = response.getHits();
SearchHit[] hits = searchHits.getHits();
if (hits.length == 0) {
return httpResponse.sendNotFound();
} else if (hits.length > 1) {
_log.warn("found more than one resource (datasourceId: " + datasourceId + " resourceId: " + resourceId + " tenantId: " + tenant.getTenantId() + " documentId: " + documentId + ")");
}
SearchHit hit = hits[0];
String source = hit.getSourceAsString();
Resources resources = _jsonFactory.fromJson(source, Resources.class);
ResourcesPayload resourcesPayload = resources.getResources();
BinaryPayload binaryPayload = resourcesPayload.getBinaries().get(0);
String data = binaryPayload.getData();
String contentType = binaryPayload.getContentType();
_manageCache(httpRequest, httpResponse);
if (contentType != null && !contentType.isBlank()) {
httpResponse.header(_CONTENT_TYPE, contentType);
}
byte[] decode = Base64.getDecoder().decode(data);
return Mono.from(httpResponse.sendByteArray(Mono.just(decode)));
});
}
Aggregations