Examples with PluginDriverDTO io.openk9.plugin.driver.manager.model.PluginDriverDTO used on opensource projects

Example 16 with PluginDriverDTO

use of io.openk9.plugin.driver.manager.model.PluginDriverDTO in project openk9 by smclab.

the class BaseSearchHTTPHandler method _toQuerySearchRequest.

private Mono<SearchResponse> _toQuerySearchRequest(Tenant tenant, List<Datasource> datasources, PluginDriverDTOList pdDTOList, SearchRequest searchRequest, HttpServerRequest httpRequest) {
    return Mono.defer(() -> {
        List<PluginDriverDTO> pluginDriverDTOList = pdDTOList.getPluginDriverDTOList();
        Map<String, List<SearchToken>> tokenTypeGroup = searchRequest.getSearchQuery().stream().collect(Collectors.groupingBy(SearchToken::getTokenType));
        List<SearchToken> datasource = tokenTypeGroup.get("DATASOURCE");
        Stream<PluginDriverDTO> documentTypeStream = pluginDriverDTOList.stream();
        if (datasource != null) {
            List<String> datasourceValues = datasource.stream().map(SearchToken::getValues).flatMap(Arrays::stream).distinct().collect(Collectors.toList());
            documentTypeStream = documentTypeStream.filter(entry -> datasourceValues.contains(entry.getName()));
        }
        List<PluginDriverDTO> documentTypeList = documentTypeStream.collect(Collectors.toList());
        QueryParser queryParser = _queryParsers.stream().reduce(QueryParser.NOTHING, QueryParser::andThen);
        return queryParser.apply(createQueryParserContext(tenant, datasources, httpRequest, tokenTypeGroup, documentTypeList)).flatMap(boolQueryBuilderConsumer -> _search.flatMapSearch(factory -> {
            long tenantId = tenant.getTenantId();
            if (documentTypeList.isEmpty()) {
                return Mono.just(SearchUtil.EMPTY_SEARCH_REQUEST);
            }
            BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
            boolQueryBuilderConsumer.accept(boolQuery);
            org.elasticsearch.action.search.SearchRequest elasticSearchQuery;
            if (datasource != null) {
                String[] indexNames = documentTypeList.stream().map(PluginDriverDTO::getName).distinct().toArray(String[]::new);
                elasticSearchQuery = factory.createSearchRequestData(tenantId, indexNames);
            } else {
                elasticSearchQuery = factory.createSearchRequestData(tenantId, "*");
            }
            SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
            searchSourceBuilder.fetchSource(includeFields(), excludeFields());
            searchSourceBuilder.query(boolQuery);
            searchSourceBuilder.trackTotalHits(true);
            return customizeSearchSourceBuilderMono(tenant, datasources, searchRequest, documentTypeList, searchSourceBuilder, elasticSearchQuery);
        }));
    });
}

Example 17 with PluginDriverDTO

use of io.openk9.plugin.driver.manager.model.PluginDriverDTO in project openk9 by smclab.

the class BaseSearchHTTPHandler method customizeSearchSourceBuilder.

protected void customizeSearchSourceBuilder(Tenant tenant, List<Datasource> datasources, SearchRequest searchRequest, List<PluginDriverDTO> documentTypeList, SearchSourceBuilder searchSourceBuilder, org.elasticsearch.action.search.SearchRequest elasticSearchQuery) {
    int[] range = searchRequest.getRange();
    if (range != null) {
        searchSourceBuilder.from(range[0]);
        searchSourceBuilder.size(range[1]);
    }
    HighlightBuilder highlightBuilder = new HighlightBuilder();
    documentTypeList.stream().map(PluginDriverDTO::getDocumentTypes).flatMap(Collection::stream).map(DocumentTypeDTO::getSearchKeywords).flatMap(Collection::stream).filter(SearchKeywordDTO::isText).map(SearchKeywordDTO::getKeyword).distinct().forEach(highlightBuilder::field);
    highlightBuilder.forceSource(true);
    highlightBuilder.tagsSchema("default");
    searchSourceBuilder.highlighter(highlightBuilder);
}

Example 18 with PluginDriverDTO

use of io.openk9.plugin.driver.manager.model.PluginDriverDTO in project openk9 by smclab.

the class ResourcesHttpHandler method _sendResource.

private Mono<Void> _sendResource(Tenant tenant, Datasource datasource, PluginDriverDTO pluginDriverDTO, long datasourceId, String documentId, String resourceId, HttpServerRequest httpRequest, HttpServerResponse httpResponse) {
    return _queryParser.apply(QueryParser.Context.of(tenant, null, null, null, httpRequest, QueryParser.QueryCondition.DEFAULT)).flatMap(consumer -> _search.search(factory -> {
        SearchRequest searchRequest = factory.createSearchRequestData(tenant.getTenantId(), pluginDriverDTO.getName());
        BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
        boolQueryBuilder.must(QueryBuilders.idsQuery().addIds(documentId));
        boolQueryBuilder.must(QueryBuilders.nestedQuery(_RESOURCES_BINARIES, QueryBuilders.matchQuery(_RESOURCES_BINARIES_ID, resourceId), ScoreMode.Max));
        SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
        consumer.accept(boolQueryBuilder);
        searchSourceBuilder.query(boolQueryBuilder);
        searchSourceBuilder.fetchSource(new String[] { _RESOURCES_BINARIES_DATA, _RESOURCES_BINARIES_CONTENT_TYPE }, null);
        return searchRequest.source(searchSourceBuilder);
    })).flatMap(response -> {
        SearchHits searchHits = response.getHits();
        SearchHit[] hits = searchHits.getHits();
        if (hits.length == 0) {
            return httpResponse.sendNotFound();
        } else if (hits.length > 1) {
            _log.warn("found more than one resource (datasourceId: " + datasourceId + " resourceId: " + resourceId + " tenantId: " + tenant.getTenantId() + " documentId: " + documentId + ")");
        }
        SearchHit hit = hits[0];
        String source = hit.getSourceAsString();
        Resources resources = _jsonFactory.fromJson(source, Resources.class);
        ResourcesPayload resourcesPayload = resources.getResources();
        BinaryPayload binaryPayload = resourcesPayload.getBinaries().get(0);
        String data = binaryPayload.getData();
        String contentType = binaryPayload.getContentType();
        _manageCache(httpRequest, httpResponse);
        if (contentType != null && !contentType.isBlank()) {
            httpResponse.header(_CONTENT_TYPE, contentType);
        }
        byte[] decode = Base64.getDecoder().decode(data);
        return Mono.from(httpResponse.sendByteArray(Mono.just(decode)));
    });
}

Example 19 with PluginDriverDTO

use of io.openk9.plugin.driver.manager.model.PluginDriverDTO in project openk9 by smclab.

the class PluginDriverDTOServiceImpl method _findDocumentType.

private PluginDriverDTO _findDocumentType(PluginDriver pluginDriver) {
    String name = pluginDriver.getName();
    List<DocumentType> documentTypeList = _documentTypeProvider.getDocumentTypeList(name);
    DocumentType defaultDocumentType = _documentTypeProvider.getDefaultDocumentType(name);
    if (documentTypeList.isEmpty() && defaultDocumentType != null) {
        documentTypeList = List.of(defaultDocumentType);
    } else if (!documentTypeList.isEmpty() && defaultDocumentType == null) {
        defaultDocumentType = documentTypeList.get(0);
    }
    List<DocumentTypeDTO> documentTypeDTOS = documentTypeList.stream().map(documentType -> DocumentTypeDTO.of(documentType.getName(), documentType.getIcon(), _wrapSearchKeywords(documentType))).collect(Collectors.toList());
    return PluginDriverDTO.of(pluginDriver.getDriverServiceName(), pluginDriver.getName(), pluginDriver.schedulerEnabled(), documentTypeDTOS, defaultDocumentType == null ? null : DocumentTypeDTO.of(defaultDocumentType.getName(), defaultDocumentType.getIcon(), _wrapSearchKeywords(defaultDocumentType)));
}

Example 20 with PluginDriverDTO

use of io.openk9.plugin.driver.manager.model.PluginDriverDTO in project openk9 by smclab.

the class BaseNerEnrichProcessor method process.

@Override
public Mono<ObjectNode> process(ObjectNode objectNode, DatasourceContext context, EnrichItem enrichItem, PluginDriverDTO pluginDriverName) {
    return Mono.defer(() -> {
        JsonNode datasourceConfiguration = _jsonFactory.fromJsonToJsonNode(enrichItem.getJsonConfig());
        if (!datasourceConfiguration.isObject()) {
            return Mono.error(new RuntimeException("jsonConfig must be an instance of ObjectNode " + datasourceConfiguration.toString()));
        }
        ObjectNode request = prepareRequestRawContent(objectNode, datasourceConfiguration.toObjectNode(), context, pluginDriverName);
        return Mono.from(_httpClient.request(getMethod(), getPath(), request.toString(), getHeaders())).retry(5).map(_jsonFactory::fromJsonToJsonNode).map(JsonNode::toObjectNode).map(jsonNodes -> {
            jsonNodes.put(Constants.TENANT_ID, context.getTenant().getTenantId());
            jsonNodes.put(Constants.DATASOURCE_ID, context.getDatasource().getDatasourceId());
            jsonNodes.put(Constants.CONTENT_ID, objectNode.get(Constants.CONTENT_ID));
            jsonNodes.put(Constants.RAW_CONTENT, objectNode.get(Constants.RAW_CONTENT));
            jsonNodes.put(Constants.INGESTION_ID, objectNode.get(Constants.INGESTION_ID));
            return jsonNodes;
        }).flatMap(this::_getEntityOrCreate).map(entities -> objectNode.set(entitiesField(), entities));
    });
}