Search in sources :

Example 1 with ElasticVindClient

use of com.rbmhtechnology.vind.elasticsearch.backend.client.ElasticVindClient in project vind by RBMHTechnology.

the class ElasticQueryBuilder method buildElasticAggregations.

private static List<AggregationBuilder> buildElasticAggregations(String name, Facet vindFacet, DocumentFactory factory, String searchContext, int minCount, int facetLimit, List<String> indexFootPrint, ElasticVindClient client, SearchSourceBuilder searchSourceStatic) {
    final String contextualizedFacetName = Stream.of(searchContext, name).filter(Objects::nonNull).collect(Collectors.joining("_"));
    final UseCase facetScope = UseCase.valueOf(vindFacet.getScope().name());
    switch(vindFacet.getType()) {
        case "TermFacet":
            final Facet.TermFacet termFacet = (Facet.TermFacet) vindFacet;
            final FieldDescriptor<?> field = factory.getField(termFacet.getFieldName());
            if (field.hasUseCase(facetScope)) {
                final String fieldName = FieldUtil.getFieldName(field, facetScope, searchContext, indexFootPrint).orElse(termFacet.getFieldName());
                final TermsAggregationBuilder termsAgg = AggregationBuilders.terms(contextualizedFacetName).field(fieldName).minDocCount(minCount).size(facetLimit);
                Optional.ofNullable(termFacet.getOption()).ifPresent(option -> setTermOptions(termsAgg, option));
                addSortToAggregation(vindFacet, searchContext, termsAgg, indexFootPrint);
                return Collections.singletonList(termsAgg);
            } else
                return Collections.emptyList();
        case "TypeFacet":
            final Facet.TypeFacet typeFacet = (Facet.TypeFacet) vindFacet;
            final TermsAggregationBuilder typeAgg = AggregationBuilders.terms(name).field(FieldUtil.TYPE).minDocCount(minCount).size(facetLimit);
            addSortToAggregation(vindFacet, searchContext, typeAgg, indexFootPrint);
            return Collections.singletonList(typeAgg);
        case "QueryFacet":
            final List<AggregationBuilder> aggs = new ArrayList<>();
            final Facet.QueryFacet queryFacet = (Facet.QueryFacet) vindFacet;
            final Filter vindFilter = queryFacet.getFilter();
            Optional.ofNullable(filterMapper(vindFilter, factory, searchContext, indexFootPrint)).ifPresent(elasticFilter -> aggs.add(AggregationBuilders.filters(contextualizedFacetName, elasticFilter)));
            return aggs;
        case "NumericRangeFacet":
            final Facet.NumericRangeFacet<?> numericRangeFacet = (Facet.NumericRangeFacet) vindFacet;
            final Optional<String> numericRangeAggField = FieldUtil.getFieldName(numericRangeFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
            if (numericRangeAggField.isPresent()) {
                final HistogramAggregationBuilder rangeAggregation = AggregationBuilders.histogram(name).keyed(true).field(numericRangeAggField.get()).interval(numericRangeFacet.getGap().doubleValue()).minDocCount(minCount);
                addSortToAggregation(vindFacet, searchContext, rangeAggregation, indexFootPrint);
                final RangeAggregationBuilder numericIntervalRangeAggregation = AggregationBuilders.range(contextualizedFacetName).keyed(true).field(numericRangeAggField.get()).subAggregation(rangeAggregation);
                numericIntervalRangeAggregation.addRange(numericRangeFacet.getStart().doubleValue(), numericRangeFacet.getEnd().doubleValue());
                return Collections.singletonList(numericIntervalRangeAggregation);
            }
            return Collections.emptyList();
        case "ZoneDateRangeFacet":
            final Facet.DateRangeFacet.ZoneDateRangeFacet zoneDateRangeFacet = (Facet.DateRangeFacet.ZoneDateRangeFacet) vindFacet;
            final Optional<String> zoneDateRangeAggFieldName = FieldUtil.getFieldName(zoneDateRangeFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
            if (zoneDateRangeAggFieldName.isPresent()) {
                final DateRangeAggregationBuilder dateRangeAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(zoneDateRangeAggFieldName.get());
                final ZonedDateTime zonedDateTimeEnd = (ZonedDateTime) zoneDateRangeFacet.getEnd();
                final ZonedDateTime zonedDateTimeStart = (ZonedDateTime) zoneDateRangeFacet.getStart();
                dateRangeAggregation.addRange(zonedDateTimeStart, zonedDateTimeEnd);
                final DateHistogramAggregationBuilder histogramDateRangeAggregation = AggregationBuilders.dateHistogram(name).minDocCount(minCount).fixedInterval(DateHistogramInterval.minutes(new Long(zoneDateRangeFacet.getGapDuration().toMinutes()).intValue())).keyed(true).field(zoneDateRangeAggFieldName.get());
                addSortToAggregation(vindFacet, searchContext, histogramDateRangeAggregation, indexFootPrint);
                dateRangeAggregation.subAggregation(histogramDateRangeAggregation);
                return Collections.singletonList(dateRangeAggregation);
            }
            return Collections.emptyList();
        case "UtilDateRangeFacet":
            final Facet.DateRangeFacet.UtilDateRangeFacet utilDateRangeFacet = (Facet.DateRangeFacet.UtilDateRangeFacet) vindFacet;
            final Optional<String> utilDateRangeAggFieldName = FieldUtil.getFieldName(utilDateRangeFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
            if (utilDateRangeAggFieldName.isPresent()) {
                final DateRangeAggregationBuilder utilDateRangeAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(utilDateRangeAggFieldName.get());
                final ZonedDateTime dateTimeEnd = ZonedDateTime.ofInstant(((Date) utilDateRangeFacet.getEnd()).toInstant(), ZoneId.of("UTC"));
                final ZonedDateTime dateTimeStart = ZonedDateTime.ofInstant(((Date) utilDateRangeFacet.getStart()).toInstant(), ZoneId.of("UTC"));
                utilDateRangeAggregation.addRange(dateTimeStart, dateTimeEnd);
                final DateHistogramAggregationBuilder histogramUtilDateRangeAggregation = AggregationBuilders.dateHistogram(name).keyed(true).fixedInterval(DateHistogramInterval.minutes(new Long(utilDateRangeFacet.getGapDuration().toMinutes()).intValue())).field(utilDateRangeAggFieldName.get());
                addSortToAggregation(vindFacet, searchContext, histogramUtilDateRangeAggregation, indexFootPrint);
                utilDateRangeAggregation.subAggregation(histogramUtilDateRangeAggregation);
                return Collections.singletonList(utilDateRangeAggregation);
            }
            return Collections.emptyList();
        case "DateMathRangeFacet":
            final Facet.DateRangeFacet.DateMathRangeFacet dateMathRangeFacet = (Facet.DateRangeFacet.DateMathRangeFacet) vindFacet;
            final Optional<String> dateMathRangeAggFieldName = FieldUtil.getFieldName(dateMathRangeFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
            if (dateMathRangeAggFieldName.isPresent()) {
                final DateRangeAggregationBuilder dateMathDateRangeAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(dateMathRangeAggFieldName.get());
                final ZonedDateTime dateMathEnd = ZonedDateTime.ofInstant(Instant.ofEpochSecond(((DateMathExpression) dateMathRangeFacet.getEnd()).getTimeStamp()), ZoneId.of("UTC"));
                final ZonedDateTime dateMathStart = ZonedDateTime.ofInstant(Instant.ofEpochSecond(((DateMathExpression) dateMathRangeFacet.getStart()).getTimeStamp()), ZoneId.of("UTC"));
                dateMathDateRangeAggregation.addRange(name, ((DateMathExpression) dateMathRangeFacet.getStart()).toElasticString(), ((DateMathExpression) dateMathRangeFacet.getEnd()).toElasticString());
                final Long minutesGap = dateMathRangeFacet.getGapDuration().toMinutes();
                final DateHistogramAggregationBuilder histogramDateMathDateRangeAggregation = AggregationBuilders.dateHistogram(name).minDocCount(minCount).fixedInterval(DateHistogramInterval.minutes(minutesGap.intValue())).keyed(true).field(dateMathRangeAggFieldName.get());
                dateMathDateRangeAggregation.subAggregation(histogramDateMathDateRangeAggregation);
                addSortToAggregation(vindFacet, searchContext, histogramDateMathDateRangeAggregation, indexFootPrint);
                return Collections.singletonList(dateMathDateRangeAggregation);
            }
            return Collections.emptyList();
        case "NumericIntervalFacet":
            final Facet.NumericIntervalFacet numericIntervalFacet = (Facet.NumericIntervalFacet) vindFacet;
            final Optional<String> numericIntervalAggFieldName = FieldUtil.getFieldName(numericIntervalFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
            if (numericIntervalAggFieldName.isPresent()) {
                final RangeAggregationBuilder numericIntervalAggregation = AggregationBuilders.range(contextualizedFacetName).keyed(true).field(numericIntervalAggFieldName.get());
                numericIntervalFacet.getIntervals().forEach(interval -> intervalToRange((NumericInterval<?>) interval, numericIntervalAggregation));
                return Collections.singletonList(numericIntervalAggregation);
            }
            return Collections.emptyList();
        case "ZoneDateTimeIntervalFacet":
            final Facet.DateIntervalFacet.ZoneDateTimeIntervalFacet zoneDateTimeIntervalFacet = (Facet.DateIntervalFacet.ZoneDateTimeIntervalFacet) vindFacet;
            final Optional<String> zoneDateTimeIntervalAggFieldName = FieldUtil.getFieldName(zoneDateTimeIntervalFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
            if (zoneDateTimeIntervalAggFieldName.isPresent()) {
                final DateRangeAggregationBuilder ZoneDateIntervalAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(zoneDateTimeIntervalAggFieldName.get());
                zoneDateTimeIntervalFacet.getIntervals().forEach(interval -> intervalToRange((Interval.ZonedDateTimeInterval<?>) interval, ZoneDateIntervalAggregation));
                return Collections.singletonList(ZoneDateIntervalAggregation);
            }
            return Collections.emptyList();
        case "UtilDateIntervalFacet":
            final Facet.DateIntervalFacet.UtilDateIntervalFacet utilDateIntervalFacet = (Facet.DateIntervalFacet.UtilDateIntervalFacet) vindFacet;
            final Optional<String> utilDateIntervalAggFieldName = FieldUtil.getFieldName(utilDateIntervalFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
            if (utilDateIntervalAggFieldName.isPresent()) {
                final DateRangeAggregationBuilder utilDateIntervalAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(utilDateIntervalAggFieldName.get());
                utilDateIntervalFacet.getIntervals().forEach(interval -> intervalToRange((Interval.UtilDateInterval<?>) interval, utilDateIntervalAggregation));
                return Collections.singletonList(utilDateIntervalAggregation);
            }
            return Collections.emptyList();
        case "ZoneDateTimeDateMathIntervalFacet":
        case "UtilDateMathIntervalFacet":
            final Facet.DateIntervalFacet dateMathIntervalFacet = (Facet.DateIntervalFacet) vindFacet;
            final Optional<String> dateMathIntervalAggFieldName = FieldUtil.getFieldName(dateMathIntervalFacet.getFieldDescriptor(), facetScope, searchContext, indexFootPrint);
            if (dateMathIntervalAggFieldName.isPresent()) {
                final DateRangeAggregationBuilder dateMathIntervalAggregation = AggregationBuilders.dateRange(contextualizedFacetName).keyed(true).field(dateMathIntervalAggFieldName.get());
                dateMathIntervalFacet.getIntervals().forEach(interval -> intervalToRange((Interval.DateMathInterval<?>) interval, dateMathIntervalAggregation));
                return Collections.singletonList(dateMathIntervalAggregation);
            }
            return Collections.emptyList();
        case "StatsDateFacet":
        case "StatsUtilDateFacet":
        case "StatsNumericFacet":
        case "StatsFacet":
            final Facet.StatsFacet statsFacet = (Facet.StatsFacet) vindFacet;
            if (String.class.isAssignableFrom(FieldUtil.getFieldType(statsFacet.getField(), facetScope))) {
                return getStringStatsAggregationBuilders(searchContext, contextualizedFacetName, facetScope, statsFacet, indexFootPrint);
            }
            return getStatsAggregationBuilders(searchContext, contextualizedFacetName, facetScope, statsFacet, indexFootPrint);
        case "PivotFacet":
            final Facet.PivotFacet pivotFacet = (Facet.PivotFacet) vindFacet;
            final int facetSize = pivotFacet.getSize().orElse(facetLimit);
            final List<TermsAggregationBuilder> termFacets = pivotFacet.getBuckets().stream().map(bucket -> Pair.of(FieldUtil.getFieldName(bucket.getLeft(), facetScope, searchContext, indexFootPrint), bucket.getRight())).filter(bucket -> bucket.getLeft().isPresent()).map(bucket -> AggregationBuilders.terms(contextualizedFacetName).field(bucket.getLeft().get()).minDocCount(minCount).size(Optional.ofNullable(bucket.getRight()).orElse(facetSize))).collect(Collectors.toList());
            termFacets.forEach(agg -> addSortToAggregation(vindFacet, searchContext, agg, indexFootPrint));
            final TermsAggregationBuilder mainBucket = termFacets.get(0);
            if (pivotFacet.getPage().isPresent()) {
                try {
                    final int fieldCardinality = getFieldCardinality(client, mainBucket.field(), searchSourceStatic);
                    final int partitions = (int) Math.ceil(fieldCardinality / Double.valueOf(mainBucket.size()));
                    mainBucket.includeExclude(new IncludeExclude(Math.toIntExact(pivotFacet.getPage().get()), partitions));
                } catch (IOException e) {
                    throw new SearchServerException("Error calculating cardinality of field" + mainBucket.field() + " for paged pivot facet: " + e.getMessage(), e);
                }
            }
            final Optional<TermsAggregationBuilder> pivotAgg = Lists.reverse(termFacets).stream().reduce((agg1, agg2) -> agg2.subAggregation(agg1));
            return Collections.singletonList(pivotAgg.orElse(null));
        default:
            throw new SearchServerException(String.format("Error mapping Vind facet to Elasticsearch aggregation: Unknown facet type %s", vindFacet.getType()));
    }
}
Also used : Arrays(java.util.Arrays) UseCase(com.rbmhtechnology.vind.model.FieldDescriptor.UseCase) SortedSet(java.util.SortedSet) Date(java.util.Date) SearchServerException(com.rbmhtechnology.vind.SearchServerException) ZonedDateTime(java.time.ZonedDateTime) RangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder) LoggerFactory(org.slf4j.LoggerFactory) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) StringUtils(org.apache.commons.lang3.StringUtils) Pair(org.apache.commons.lang3.tuple.Pair) FulltextSearch(com.rbmhtechnology.vind.api.query.FulltextSearch) Facet(com.rbmhtechnology.vind.api.query.facet.Facet) Locale(java.util.Locale) Map(java.util.Map) UpdateOperation(com.rbmhtechnology.vind.api.query.update.UpdateOperation) SearchResponse(org.elasticsearch.action.search.SearchResponse) GeoPoint(org.elasticsearch.common.geo.GeoPoint) ExtendedStatsAggregationBuilder(org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder) BucketOrder(org.elasticsearch.search.aggregations.BucketOrder) DateHistogramInterval(org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval) CursorUtils.fromSearchAfterCursor(com.rbmhtechnology.vind.elasticsearch.backend.util.CursorUtils.fromSearchAfterCursor) Slice(com.rbmhtechnology.vind.api.query.division.Slice) Collection(java.util.Collection) Sort(com.rbmhtechnology.vind.api.query.sort.Sort) ParsedCardinality(org.elasticsearch.search.aggregations.metrics.ParsedCardinality) Streams(com.google.common.collect.Streams) DescriptorSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.DescriptorSuggestionSearch) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) ZoneId(java.time.ZoneId) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) MultiMatchQueryBuilder(org.elasticsearch.index.query.MultiMatchQueryBuilder) SuggestBuilders(org.elasticsearch.search.suggest.SuggestBuilders) Optional(java.util.Optional) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) ElasticVindClient(com.rbmhtechnology.vind.elasticsearch.backend.client.ElasticVindClient) Option(org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option) TermFacetOption(com.rbmhtechnology.vind.api.query.facet.TermFacetOption) Interval(com.rbmhtechnology.vind.api.query.facet.Interval) IncludeExclude(org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude) DateRangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder) AggregationBuilder(org.elasticsearch.search.aggregations.AggregationBuilder) Suggest(org.elasticsearch.search.suggest.Suggest) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) NumericInterval(com.rbmhtechnology.vind.api.query.facet.Interval.NumericInterval) ExecutableSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.ExecutableSuggestionSearch) ScriptType(org.elasticsearch.script.ScriptType) FieldDescriptor(com.rbmhtechnology.vind.model.FieldDescriptor) ArrayUtils(org.apache.commons.lang3.ArrayUtils) HashMap(java.util.HashMap) FulltextTerm(com.rbmhtechnology.vind.api.query.FulltextTerm) Strings(org.elasticsearch.common.Strings) ArrayList(java.util.ArrayList) HighlightBuilder(org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder) Lists(com.google.common.collect.Lists) Operator(org.elasticsearch.index.query.Operator) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) DateMathExpression(com.rbmhtechnology.vind.api.query.datemath.DateMathExpression) TermsAggregationBuilder(org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder) SearchConfiguration(com.rbmhtechnology.vind.configure.SearchConfiguration) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Script(org.elasticsearch.script.Script) Logger(org.slf4j.Logger) StringSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.StringSuggestionSearch) AggregationBuilders(org.elasticsearch.search.aggregations.AggregationBuilders) IOException(java.io.IOException) HistogramAggregationBuilder(org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder) SuggestBuilder(org.elasticsearch.search.suggest.SuggestBuilder) DisMaxQueryBuilder(org.elasticsearch.index.query.DisMaxQueryBuilder) DateHistogramAggregationBuilder(org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder) DistanceUnit(org.elasticsearch.common.unit.DistanceUnit) DocumentFactory(com.rbmhtechnology.vind.model.DocumentFactory) Cursor(com.rbmhtechnology.vind.api.query.division.Cursor) Page(com.rbmhtechnology.vind.api.query.division.Page) Filter(com.rbmhtechnology.vind.api.query.filter.Filter) Comparator(java.util.Comparator) Collections(java.util.Collections) NUMBER_OF_MATCHING_TERMS_SORT(com.rbmhtechnology.vind.elasticsearch.backend.util.SortUtils.NUMBER_OF_MATCHING_TERMS_SORT) RangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder) ExtendedStatsAggregationBuilder(org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder) DateRangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder) AggregationBuilder(org.elasticsearch.search.aggregations.AggregationBuilder) TermsAggregationBuilder(org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder) HistogramAggregationBuilder(org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder) DateHistogramAggregationBuilder(org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder) ArrayList(java.util.ArrayList) NumericInterval(com.rbmhtechnology.vind.api.query.facet.Interval.NumericInterval) ZonedDateTime(java.time.ZonedDateTime) SearchServerException(com.rbmhtechnology.vind.SearchServerException) Facet(com.rbmhtechnology.vind.api.query.facet.Facet) HistogramAggregationBuilder(org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder) DateHistogramAggregationBuilder(org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder) DateHistogramAggregationBuilder(org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder) DateRangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder) TermsAggregationBuilder(org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder) RangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder) DateRangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder) IncludeExclude(org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude) UseCase(com.rbmhtechnology.vind.model.FieldDescriptor.UseCase) IOException(java.io.IOException) GeoPoint(org.elasticsearch.common.geo.GeoPoint) Filter(com.rbmhtechnology.vind.api.query.filter.Filter) DateMathExpression(com.rbmhtechnology.vind.api.query.datemath.DateMathExpression)

Example 2 with ElasticVindClient

use of com.rbmhtechnology.vind.elasticsearch.backend.client.ElasticVindClient in project vind by RBMHTechnology.

the class ElasticQueryBuilder method buildQuery.

public static SearchSourceBuilder buildQuery(FulltextSearch search, DocumentFactory factory, boolean escape, List<String> indexFootPrint, ElasticVindClient client) {
    final String searchContext = search.getSearchContext();
    final SearchSourceBuilder searchSource = new SearchSourceBuilder();
    final BoolQueryBuilder baseQuery = QueryBuilders.boolQuery();
    // Set total hits count
    final boolean trackTotalHits = Boolean.parseBoolean(SearchConfiguration.get(SearchConfiguration.TRACK_TOTAL_HITS, "true"));
    searchSource.trackTotalHits(trackTotalHits);
    // build full text disMax query
    String searchString = "*".equals(search.getSearchString()) || Strings.isEmpty(search.getSearchString().trim()) ? "*:*" : search.getSearchString().trim();
    if (escape) {
        // Escape especial characters: + - = && || > < ! ( ) { } [ ] ^ " ~ * ? : \ /
        for (Map.Entry<String, String> wordEntry : reservedChars.entrySet()) {
            searchString = searchString.replaceAll(wordEntry.getKey(), wordEntry.getValue());
        }
    }
    if (!escape && searchString.contains(":")) {
        String skipColonSearchString = searchString.replaceAll(":", reservedChars.get(":"));
        final String[] skipedFields = Arrays.stream(skipColonSearchString.split(" ")).filter(term -> term.contains("\\:")).map(term -> term.substring(0, term.indexOf("\\:"))).filter(posibleField -> indexFootPrint.contains(posibleField) || "*".equals(posibleField)).toArray(String[]::new);
        for (String field : skipedFields) {
            skipColonSearchString = skipColonSearchString.replace(field + "\\:", field + ":");
        }
        searchString = skipColonSearchString;
    }
    final DisMaxQueryBuilder query = createDisMaxQueryBuilder(new FulltextTerm(searchString, search.getMinimumShouldMatch()), factory, indexFootPrint, searchContext);
    baseQuery.must(query);
    searchSource.query(baseQuery);
    if (search.isSpellcheck()) {
        final SuggestBuilder suggestBuilder = new SuggestBuilder();
        suggestBuilder.setGlobalText(search.getSearchString());
        Lists.newArrayList(getFullTextFieldNames(search, factory, searchContext, indexFootPrint)).forEach(fieldName -> suggestBuilder.addSuggestion(FieldUtil.getSourceFieldName(fieldName.replaceAll(".text", ""), searchContext), SuggestBuilders.termSuggestion(fieldName).prefixLength(0)));
        searchSource.suggest(suggestBuilder);
    }
    searchSource.trackScores(SearchConfiguration.get(SearchConfiguration.SEARCH_RESULT_SHOW_SCORE, true));
    if (search.getGeoDistance() != null) {
        final FieldDescriptor distanceField = factory.getField(search.getGeoDistance().getFieldName());
        final Optional<String> distanceFieldName = FieldUtil.getFieldName(distanceField, searchContext, indexFootPrint);
        if (Objects.nonNull(distanceField) && distanceFieldName.isPresent()) {
            searchSource.scriptField(FieldUtil.DISTANCE, new Script(ScriptType.INLINE, "painless", String.format(Locale.ENGLISH, "if(doc['%s'].size()!=0)" + "doc['%s'].arcDistance(%f,%f);" + "else []", distanceFieldName.get(), distanceFieldName.get(), search.getGeoDistance().getLocation().getLat(), search.getGeoDistance().getLocation().getLng()), Collections.emptyMap()));
        }
    }
    searchSource.fetchSource(true);
    baseQuery.filter(buildFilterQuery(search.getFilter(), factory, searchContext, indexFootPrint));
    if (search.hasFacet() && search.getFacetLimit() != 0) {
        final int facetLimit = search.getFacetLimit() < 0 ? Integer.MAX_VALUE : search.getFacetLimit();
        if (search.getFacetLimit() < 0) {
            log.info("Facet limit has been set to {}: " + "Elastic search does not support unlimited facet results, setting it to Integer.MAX_VALUE ({})", search.getFacetLimit(), facetLimit);
        }
        search.getFacets().entrySet().stream().map(vindFacet -> {
            return buildElasticAggregations(vindFacet.getKey(), vindFacet.getValue(), factory, searchContext, search.getFacetMinCount(), facetLimit, indexFootPrint, client, searchSource);
        }).flatMap(Collection::stream).filter(Objects::nonNull).forEach(searchSource::aggregation);
    }
    search.getFacets().values().stream().filter(facet -> facet.getType().equals("PivotFacet")).map(pivotFacet -> searchSource.aggregations().getAggregatorFactories().stream().filter(agg -> agg.getName().equals(Stream.of(searchContext, pivotFacet.getFacetName()).filter(Objects::nonNull).collect(Collectors.joining("_")))).collect(Collectors.toList())).flatMap(Collection::stream).forEach(pivotAgg -> {
        final List<String> facets = search.getFacets().values().stream().filter(facet -> Arrays.asList(facet.getTagedPivots()).contains(pivotAgg.getName().replaceAll(searchContext + "_", ""))).map(facet -> Stream.of(searchContext, facet.getFacetName()).filter(Objects::nonNull).collect(Collectors.joining("_"))).collect(Collectors.toList());
        final List<AggregationBuilder> aggs = searchSource.aggregations().getAggregatorFactories().stream().filter(agg -> facets.contains(agg.getName())).collect(Collectors.toList());
        addToPivotAggs(pivotAgg, aggs, indexFootPrint);
    });
    // sorting
    if (search.hasSorting()) {
        search.getSorting().stream().map(sort -> SortUtils.buildSort(sort, search, factory, searchContext, indexFootPrint)).forEach(searchSource::sort);
    }
    // paging
    switch(search.getResultSet().getType()) {
        case page:
            {
                final Page resultSet = (Page) search.getResultSet();
                searchSource.from(resultSet.getOffset());
                searchSource.size(resultSet.getPagesize());
                break;
            }
        case slice:
            {
                final Slice resultSet = (Slice) search.getResultSet();
                searchSource.from(resultSet.getOffset());
                searchSource.size(resultSet.getSliceSize());
                break;
            }
        case cursor:
            {
                final Cursor resultSet = (Cursor) search.getResultSet();
                searchSource.size(resultSet.getSize());
                searchSource.sort("_id_");
                if (resultSet.getSearchAfter() != null) {
                    searchSource.searchAfter(fromSearchAfterCursor(resultSet.getSearchAfter()));
                }
                break;
            }
    }
    return searchSource;
}
Also used : Arrays(java.util.Arrays) UseCase(com.rbmhtechnology.vind.model.FieldDescriptor.UseCase) SortedSet(java.util.SortedSet) Date(java.util.Date) SearchServerException(com.rbmhtechnology.vind.SearchServerException) ZonedDateTime(java.time.ZonedDateTime) RangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder) LoggerFactory(org.slf4j.LoggerFactory) QueryBuilders(org.elasticsearch.index.query.QueryBuilders) StringUtils(org.apache.commons.lang3.StringUtils) Pair(org.apache.commons.lang3.tuple.Pair) FulltextSearch(com.rbmhtechnology.vind.api.query.FulltextSearch) Facet(com.rbmhtechnology.vind.api.query.facet.Facet) Locale(java.util.Locale) Map(java.util.Map) UpdateOperation(com.rbmhtechnology.vind.api.query.update.UpdateOperation) SearchResponse(org.elasticsearch.action.search.SearchResponse) GeoPoint(org.elasticsearch.common.geo.GeoPoint) ExtendedStatsAggregationBuilder(org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder) BucketOrder(org.elasticsearch.search.aggregations.BucketOrder) DateHistogramInterval(org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramInterval) CursorUtils.fromSearchAfterCursor(com.rbmhtechnology.vind.elasticsearch.backend.util.CursorUtils.fromSearchAfterCursor) Slice(com.rbmhtechnology.vind.api.query.division.Slice) Collection(java.util.Collection) Sort(com.rbmhtechnology.vind.api.query.sort.Sort) ParsedCardinality(org.elasticsearch.search.aggregations.metrics.ParsedCardinality) Streams(com.google.common.collect.Streams) DescriptorSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.DescriptorSuggestionSearch) Instant(java.time.Instant) Collectors(java.util.stream.Collectors) ZoneId(java.time.ZoneId) Objects(java.util.Objects) List(java.util.List) Stream(java.util.stream.Stream) MultiMatchQueryBuilder(org.elasticsearch.index.query.MultiMatchQueryBuilder) SuggestBuilders(org.elasticsearch.search.suggest.SuggestBuilders) Optional(java.util.Optional) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) ElasticVindClient(com.rbmhtechnology.vind.elasticsearch.backend.client.ElasticVindClient) Option(org.elasticsearch.search.suggest.Suggest.Suggestion.Entry.Option) TermFacetOption(com.rbmhtechnology.vind.api.query.facet.TermFacetOption) Interval(com.rbmhtechnology.vind.api.query.facet.Interval) IncludeExclude(org.elasticsearch.search.aggregations.bucket.terms.IncludeExclude) DateRangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder) AggregationBuilder(org.elasticsearch.search.aggregations.AggregationBuilder) Suggest(org.elasticsearch.search.suggest.Suggest) QueryStringQueryBuilder(org.elasticsearch.index.query.QueryStringQueryBuilder) NumericInterval(com.rbmhtechnology.vind.api.query.facet.Interval.NumericInterval) ExecutableSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.ExecutableSuggestionSearch) ScriptType(org.elasticsearch.script.ScriptType) FieldDescriptor(com.rbmhtechnology.vind.model.FieldDescriptor) ArrayUtils(org.apache.commons.lang3.ArrayUtils) HashMap(java.util.HashMap) FulltextTerm(com.rbmhtechnology.vind.api.query.FulltextTerm) Strings(org.elasticsearch.common.Strings) ArrayList(java.util.ArrayList) HighlightBuilder(org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder) Lists(com.google.common.collect.Lists) Operator(org.elasticsearch.index.query.Operator) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) DateMathExpression(com.rbmhtechnology.vind.api.query.datemath.DateMathExpression) TermsAggregationBuilder(org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder) SearchConfiguration(com.rbmhtechnology.vind.configure.SearchConfiguration) QueryBuilder(org.elasticsearch.index.query.QueryBuilder) Script(org.elasticsearch.script.Script) Logger(org.slf4j.Logger) StringSuggestionSearch(com.rbmhtechnology.vind.api.query.suggestion.StringSuggestionSearch) AggregationBuilders(org.elasticsearch.search.aggregations.AggregationBuilders) IOException(java.io.IOException) HistogramAggregationBuilder(org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder) SuggestBuilder(org.elasticsearch.search.suggest.SuggestBuilder) DisMaxQueryBuilder(org.elasticsearch.index.query.DisMaxQueryBuilder) DateHistogramAggregationBuilder(org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder) DistanceUnit(org.elasticsearch.common.unit.DistanceUnit) DocumentFactory(com.rbmhtechnology.vind.model.DocumentFactory) Cursor(com.rbmhtechnology.vind.api.query.division.Cursor) Page(com.rbmhtechnology.vind.api.query.division.Page) Filter(com.rbmhtechnology.vind.api.query.filter.Filter) Comparator(java.util.Comparator) Collections(java.util.Collections) NUMBER_OF_MATCHING_TERMS_SORT(com.rbmhtechnology.vind.elasticsearch.backend.util.SortUtils.NUMBER_OF_MATCHING_TERMS_SORT) Script(org.elasticsearch.script.Script) RangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.RangeAggregationBuilder) ExtendedStatsAggregationBuilder(org.elasticsearch.search.aggregations.metrics.ExtendedStatsAggregationBuilder) DateRangeAggregationBuilder(org.elasticsearch.search.aggregations.bucket.range.DateRangeAggregationBuilder) AggregationBuilder(org.elasticsearch.search.aggregations.AggregationBuilder) TermsAggregationBuilder(org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder) HistogramAggregationBuilder(org.elasticsearch.search.aggregations.bucket.histogram.HistogramAggregationBuilder) DateHistogramAggregationBuilder(org.elasticsearch.search.aggregations.bucket.histogram.DateHistogramAggregationBuilder) FulltextTerm(com.rbmhtechnology.vind.api.query.FulltextTerm) Page(com.rbmhtechnology.vind.api.query.division.Page) CursorUtils.fromSearchAfterCursor(com.rbmhtechnology.vind.elasticsearch.backend.util.CursorUtils.fromSearchAfterCursor) Cursor(com.rbmhtechnology.vind.api.query.division.Cursor) GeoPoint(org.elasticsearch.common.geo.GeoPoint) SearchSourceBuilder(org.elasticsearch.search.builder.SearchSourceBuilder) FieldDescriptor(com.rbmhtechnology.vind.model.FieldDescriptor) SuggestBuilder(org.elasticsearch.search.suggest.SuggestBuilder) BoolQueryBuilder(org.elasticsearch.index.query.BoolQueryBuilder) DisMaxQueryBuilder(org.elasticsearch.index.query.DisMaxQueryBuilder) Slice(com.rbmhtechnology.vind.api.query.division.Slice) Collection(java.util.Collection) Map(java.util.Map) HashMap(java.util.HashMap)

Example 3 with ElasticVindClient

use of com.rbmhtechnology.vind.elasticsearch.backend.client.ElasticVindClient in project vind by RBMHTechnology.

the class ElasticsearchServerProvider method getInstance.

@Override
public ElasticVindClient getInstance() {
    final Logger log = LoggerFactory.getLogger(ElasticServerProvider.class);
    final String host = SearchConfiguration.get(SearchConfiguration.SERVER_HOST);
    if (host == null) {
        log.error("{} has to be set", SearchConfiguration.SERVER_HOST);
        throw new SearchServerInstantiateException(SearchConfiguration.SERVER_HOST + " has to be set", this.getClass());
    }
    final String collection = SearchConfiguration.get(SearchConfiguration.SERVER_COLLECTION);
    final String connectionTimeout = SearchConfiguration.get(SearchConfiguration.SERVER_CONNECTION_TIMEOUT);
    final String soTimeout = SearchConfiguration.get(SearchConfiguration.SERVER_SO_TIMEOUT);
    log.info("Instantiating Elasticsearch client: {}", host);
    if (collection != null) {
        ElasticVindClient client;
        final AuthTypes authType = AuthTypes.valueOf(SearchConfiguration.get(SearchConfiguration.SEARCH_AUTHENTICATION_METHOD, AuthTypes.NONE.name()));
        final ElasticVindClient.Builder clientBuilder = new ElasticVindClient.Builder(host).setDefaultIndex(collection);
        if (StringUtils.isNotEmpty(connectionTimeout)) {
            clientBuilder.setConnectionTimeout(Long.parseLong(connectionTimeout));
        }
        if (StringUtils.isNotEmpty(soTimeout)) {
            clientBuilder.setSocketTimeout(Long.parseLong(soTimeout));
        }
        switch(authType) {
            case APIKEY:
                final String id = SearchConfiguration.get(SearchConfiguration.SEARCH_API_KEY_ID);
                final String key = SearchConfiguration.get(SearchConfiguration.SEARCH_API_KEY_SECRET);
                if (Objects.isNull(id) || Objects.isNull(key)) {
                    log.error("Missing API id or secret to authenticate with Elasticsearch backend");
                    throw new SearchServerInstantiateException("Missing API id or secret to authenticate with Elasticsearch backend", this.getClass());
                }
                client = clientBuilder.buildWithApiKeyAuth(id, key);
                break;
            case BASIC:
                final String user = SearchConfiguration.get(SearchConfiguration.SEARCH_AUTHENTICATION_USER);
                final String pssw = SearchConfiguration.get(SearchConfiguration.SEARCH_AUTHENTICATION_KEY);
                if (Objects.isNull(user) || Objects.isNull(pssw)) {
                    log.error("Missing API user or password to authenticate with Elasticsearch backend");
                    throw new SearchServerInstantiateException("Missing API user or password to authenticate with Elasticsearch backend", this.getClass());
                }
                client = clientBuilder.buildWithBasicAuth(user, pssw);
                break;
            default:
                client = clientBuilder.build();
                break;
        }
        if (StringUtils.isNotEmpty(connectionTimeout)) {
            client.setConnectionTimeout(Long.parseLong(connectionTimeout));
        }
        if (StringUtils.isNotEmpty(soTimeout)) {
            client.setClientTimeout(Long.parseLong(soTimeout));
        }
        return client;
    } else {
        log.error(SearchConfiguration.SERVER_COLLECTION + " has to be set");
        throw new SearchServerInstantiateException(SearchConfiguration.SERVER_COLLECTION + " has to be set", this.getClass());
    }
}
Also used : ElasticVindClient(com.rbmhtechnology.vind.elasticsearch.backend.client.ElasticVindClient) SearchServerInstantiateException(com.rbmhtechnology.vind.SearchServerInstantiateException) Logger(org.slf4j.Logger)

Aggregations

ElasticVindClient (com.rbmhtechnology.vind.elasticsearch.backend.client.ElasticVindClient)3 Lists (com.google.common.collect.Lists)2 Streams (com.google.common.collect.Streams)2 SearchServerException (com.rbmhtechnology.vind.SearchServerException)2 FulltextSearch (com.rbmhtechnology.vind.api.query.FulltextSearch)2 FulltextTerm (com.rbmhtechnology.vind.api.query.FulltextTerm)2 DateMathExpression (com.rbmhtechnology.vind.api.query.datemath.DateMathExpression)2 Cursor (com.rbmhtechnology.vind.api.query.division.Cursor)2 Page (com.rbmhtechnology.vind.api.query.division.Page)2 Slice (com.rbmhtechnology.vind.api.query.division.Slice)2 Facet (com.rbmhtechnology.vind.api.query.facet.Facet)2 Interval (com.rbmhtechnology.vind.api.query.facet.Interval)2 NumericInterval (com.rbmhtechnology.vind.api.query.facet.Interval.NumericInterval)2 TermFacetOption (com.rbmhtechnology.vind.api.query.facet.TermFacetOption)2 Filter (com.rbmhtechnology.vind.api.query.filter.Filter)2 Sort (com.rbmhtechnology.vind.api.query.sort.Sort)2 DescriptorSuggestionSearch (com.rbmhtechnology.vind.api.query.suggestion.DescriptorSuggestionSearch)2 ExecutableSuggestionSearch (com.rbmhtechnology.vind.api.query.suggestion.ExecutableSuggestionSearch)2 StringSuggestionSearch (com.rbmhtechnology.vind.api.query.suggestion.StringSuggestionSearch)2 UpdateOperation (com.rbmhtechnology.vind.api.query.update.UpdateOperation)2