Examples with Field io.openk9.plugin.driver.manager.api.Field used on opensource projects

Example 1 with Field

use of io.openk9.plugin.driver.manager.api.Field in project openk9 by smclab.

the class SuggestionsHTTPHandler method customizeSearchSourceBuilderMono.

@Override
protected Mono<org.elasticsearch.action.search.SearchRequest> customizeSearchSourceBuilderMono(Tenant tenant, List<Datasource> datasources, SearchRequest searchRequest, List<PluginDriverDTO> documentTypeList, SearchSourceBuilder searchSourceBuilder, org.elasticsearch.action.search.SearchRequest elasticSearchQuery) {
    return Mono.defer(() -> {
        Long suggestionCategoryId = searchRequest.getSuggestionCategoryId();
        if (suggestionCategoryId == null) {
            return _datasourceClient.findSuggestionCategoryFieldsByTenantId(tenant.getTenantId());
        } else {
            return _datasourceClient.findSuggestionCategoryFieldsByTenantIdAndCategoryId(tenant.getTenantId(), suggestionCategoryId);
        }
    }).map(fields -> {
        if (!(fields == null || fields.isEmpty())) {
            Function<String, CompositeValuesSourceBuilder<?>> fieldToTerms = nameField -> new TermsValuesSourceBuilder(nameField).field(nameField).missingBucket(true);
            CompositeAggregationBuilder compositeAggregation = fields.stream().map(SuggestionCategoryField::getFieldName).map(fieldToTerms).collect(Collectors.collectingAndThen(Collectors.toList(), list -> AggregationBuilders.composite("composite", list)));
            String afterKey = searchRequest.getAfterKey();
            if (afterKey != null) {
                byte[] afterKeyDecoded = Base64.getDecoder().decode(afterKey);
                Map<String, Object> map = _jsonFactory.fromJsonMap(new String(afterKeyDecoded), Object.class);
                compositeAggregation.aggregateAfter(map);
            }
            int[] range = searchRequest.getRange();
            if (range != null && range.length == 2) {
                int size = range[1];
                compositeAggregation.size(size);
            }
            searchSourceBuilder.aggregation(compositeAggregation);
        }
        searchSourceBuilder.from(0);
        searchSourceBuilder.size(0);
        searchSourceBuilder.highlighter(null);
        return elasticSearchQuery.source(searchSourceBuilder);
    });
}

Example 2 with Field

use of io.openk9.plugin.driver.manager.api.Field in project openk9 by smclab.

the class BaseAggregatorAnnotator method annotate_.

@Override
public List<CategorySemantics> annotate_(long tenantId, String... tokens) {
    List<String> normalizedKeywords = tenantKeywordsMap.getOrDefault(tenantId, tenantKeywordsMap.get(-1L));
    if (normalizedKeywords == null) {
        return List.of();
    }
    RestHighLevelClient restHighLevelClient = restHighLevelClientProvider.get();
    String token;
    if (tokens.length == 1) {
        token = tokens[0];
    } else {
        token = String.join(" ", tokens);
    }
    BoolQueryBuilder builder = QueryBuilders.boolQuery();
    BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
    for (String keyword : normalizedKeywords) {
        boolQueryBuilder.should(query(keyword, token));
    }
    builder.must(boolQueryBuilder);
    SearchRequest searchRequest;
    if (tenantId == -1) {
        searchRequest = new SearchRequest("*-*-data");
    } else {
        searchRequest = new SearchRequest(tenantId + "-*-data");
    }
    SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    searchSourceBuilder.size(0);
    searchSourceBuilder.query(builder);
    for (String keyword : normalizedKeywords) {
        searchSourceBuilder.aggregation(AggregationBuilders.terms(keyword).field(keyword).size(10));
    }
    searchRequest.source(searchSourceBuilder);
    if (_log.isDebugEnabled()) {
        _log.debug(builder.toString());
    }
    List<Tuple> scoreKeys = new ArrayList<>();
    try {
        SearchResponse search = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
        for (Aggregation aggregation : search.getAggregations()) {
            Terms terms = (Terms) aggregation;
            for (Terms.Bucket bucket : terms.getBuckets()) {
                String keyAsString = bucket.getKeyAsString();
                if (token.equalsIgnoreCase(keyAsString)) {
                    return List.of(_createCategorySemantics(terms.getName(), keyAsString));
                }
                scoreKeys.add(Tuple.of((Supplier<Double>) () -> _levenshteinDistance(token, keyAsString), keyAsString, terms.getName()));
            }
        }
    } catch (IOException e) {
        _log.error(e.getMessage(), e);
    }
    if (scoreKeys.isEmpty()) {
        return List.of();
    }
    scoreKeys.sort(Collections.reverseOrder(Comparator.comparingDouble(t -> ((Supplier<Double>) t.get(0)).get())));
    String key = (String) scoreKeys.get(0).get(1);
    String name = (String) scoreKeys.get(0).get(2);
    return List.of(_createCategorySemantics(name, key));
}

Example 3 with Field

use of io.openk9.plugin.driver.manager.api.Field in project openk9 by smclab.

the class EnrichPipelineProcessor method _adaptIngestionPayload.

private Tuple3<DatasourceContext, ObjectNode, PluginDriverDTO> _adaptIngestionPayload(DatasourceContext datasourceContext, ObjectNode ingestionPayload, PluginDriverDTO pluginDriverDTO) {
    ObjectNode newIngestionPayload = ingestionPayload;
    if (newIngestionPayload.hasNonNull(io.openk9.core.api.constant.Constants.DATASOURCE_PAYLOAD)) {
        JsonNode datasourcePayload = newIngestionPayload.remove(io.openk9.core.api.constant.Constants.DATASOURCE_PAYLOAD);
        if (datasourcePayload.isObject()) {
            ObjectNode jsonNodes = datasourcePayload.toObjectNode();
            for (Map.Entry<String, JsonNode> field : jsonNodes.fields()) {
                newIngestionPayload.set(field.getKey(), field.getValue());
            }
        }
    }
    if (!ingestionPayload.hasNonNull(Constants.DATASOURCE_NAME)) {
        newIngestionPayload = newIngestionPayload.deepCopy().put(Constants.DATASOURCE_NAME, pluginDriverDTO.getName());
    }
    if (ingestionPayload.hasNonNull(io.openk9.core.api.constant.Constants.TYPE) && !ingestionPayload.get(io.openk9.core.api.constant.Constants.TYPE).toArrayNode().isEmpty()) {
        return Tuples.of(datasourceContext, ingestionPayload, pluginDriverDTO);
    }
    DocumentTypeDTO documentType = pluginDriverDTO.getDefaultDocumentType();
    return Tuples.of(datasourceContext, newIngestionPayload.set(io.openk9.core.api.constant.Constants.TYPE, _jsonFactory.createArrayNode().add(documentType.getName())), pluginDriverDTO);
}

Example 4 with Field

use of io.openk9.plugin.driver.manager.api.Field in project openk9 by smclab.

the class BaseSearchHTTPHandler method customizeSearchSourceBuilder.

protected void customizeSearchSourceBuilder(Tenant tenant, List<Datasource> datasources, SearchRequest searchRequest, List<PluginDriverDTO> documentTypeList, SearchSourceBuilder searchSourceBuilder, org.elasticsearch.action.search.SearchRequest elasticSearchQuery) {
    int[] range = searchRequest.getRange();
    if (range != null) {
        searchSourceBuilder.from(range[0]);
        searchSourceBuilder.size(range[1]);
    }
    HighlightBuilder highlightBuilder = new HighlightBuilder();
    documentTypeList.stream().map(PluginDriverDTO::getDocumentTypes).flatMap(Collection::stream).map(DocumentTypeDTO::getSearchKeywords).flatMap(Collection::stream).filter(SearchKeywordDTO::isText).map(SearchKeywordDTO::getKeyword).distinct().forEach(highlightBuilder::field);
    highlightBuilder.forceSource(true);
    highlightBuilder.tagsSchema("default");
    searchSourceBuilder.highlighter(highlightBuilder);
}

Example 5 with Field

use of io.openk9.plugin.driver.manager.api.Field in project openk9 by smclab.

the class MappingsDocumentTypeFactoryCustomizer method apply.

@Override
public Mono<Void> apply(Map.Entry<String, List<DocumentType>> entry) {
    List<Field> collect = entry.getValue().stream().flatMap(documentType -> {
        List<Field> sourceFields = documentType.getSourceFields();
        if (sourceFields == null) {
            return Stream.empty();
        }
        if (sourceFields.size() == 1) {
            Field field = sourceFields.get(0);
            if (field instanceof Field.FieldMappings) {
                return Stream.of(field);
            }
        }
        return sourceFields.stream().map(child -> Field.of(documentType.getName(), child));
    }).collect(Collectors.toList());
    if (collect.stream().allMatch(f -> f instanceof Field.FieldMappings)) {
        Map<String, Object> accumulator = new HashMap<>();
        for (Field field : collect) {
            deepMerge(accumulator, field.getExtra());
        }
        String pluginDriverName = entry.getKey();
        return _indexWriterEventPublisher.publishCreateIndexTemplate(IndexTemplateDTO.of(pluginDriverName + "_template", null, List.of("*-" + pluginDriverName + "-data"), _jsonFactory.toJson(accumulator), List.of("data"), 10));
    }
    Map<String, Object> objectNode = new HashMap<>();
    for (Field parent : collect) {
        Map<String, Object> fieldNode = _createFieldNode(parent);
        Map<String, Object> parentNodeWithName = fieldNode;
        Field child = parent.getChild();
        while (child != Field.FieldObj.NIL) {
            Map<String, Object> parentNode = (Map<String, Object>) parentNodeWithName.get(parent.getName());
            Map<String, Object> childNode = _createFieldNode(child);
            parentNode.put("properties", childNode);
            parentNodeWithName = childNode;
            parent = child;
            child = child.getChild();
        }
        objectNode = _merge(objectNode, fieldNode);
    }
    String pluginDriverName = entry.getKey();
    return _indexWriterEventPublisher.publishCreateIndexTemplate(IndexTemplateDTO.of(pluginDriverName + "_template", null, List.of("*-" + pluginDriverName + "-data"), _jsonFactory.toJson(Map.of("properties", objectNode)), List.of("data"), 10));
}