Examples with SearchSourceBuilder org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder used on opensource projects

Example 71 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project graylog2-server by Graylog2.

the class IndexToolsAdapterES6 method count.

@Override
public long count(Set<String> indices, Optional<Set<String>> includedStreams) {
    final SearchSourceBuilder queryBuilder = new SearchSourceBuilder().query(buildStreamIdFilter(includedStreams));
    final Count.Builder builder = new Count.Builder().query(queryBuilder.toString()).addIndex(indices).addType(IndexMapping.TYPE_MESSAGE).setParameter(Parameters.IGNORE_UNAVAILABLE, true);
    final CountResult result = JestUtils.execute(jestClient, builder.build(), () -> "Unable to count documents of index.");
    return result.getCount().longValue();
}

Example 72 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project graylog2-server by Graylog2.

the class IndexToolsAdapterES6 method fieldHistogram.

@Override
public Map<DateTime, Map<String, Long>> fieldHistogram(String fieldName, Set<String> indices, Optional<Set<String>> includedStreams, long interval) {
    final BoolQueryBuilder queryBuilder = buildStreamIdFilter(includedStreams);
    final FilterAggregationBuilder the_filter = AggregationBuilders.filter(AGG_FILTER, queryBuilder).subAggregation(AggregationBuilders.dateHistogram(AGG_DATE_HISTOGRAM).field("timestamp").subAggregation(AggregationBuilders.terms(AGG_MESSAGE_FIELD).field(fieldName)).interval(interval).minDocCount(1L));
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).aggregation(the_filter);
    final Search.Builder searchBuilder = new Search.Builder(searchSourceBuilder.toString()).addIndex(indices).addType(IndexMapping.TYPE_MESSAGE);
    final SearchResult searchResult = JestUtils.execute(this.jestClient, searchBuilder.build(), () -> "Unable to retrieve field histogram.");
    final FilterAggregation filterAggregation = searchResult.getAggregations().getFilterAggregation(AGG_FILTER);
    final DateHistogramAggregation dateHistogram = filterAggregation.getDateHistogramAggregation(AGG_DATE_HISTOGRAM);
    final List<DateHistogramAggregation.DateHistogram> histogramBuckets = dateHistogram.getBuckets();
    final Map<DateTime, Map<String, Long>> result = Maps.newHashMapWithExpectedSize(histogramBuckets.size());
    for (HistogramAggregation.Histogram bucket : histogramBuckets) {
        final DateTime date = new DateTime(bucket.getKey()).toDateTime(DateTimeZone.UTC);
        final TermsAggregation sourceFieldAgg = bucket.getTermsAggregation(AGG_MESSAGE_FIELD);
        final List<TermsAggregation.Entry> termBuckets = sourceFieldAgg.getBuckets();
        final HashMap<String, Long> termCounts = Maps.newHashMapWithExpectedSize(termBuckets.size());
        for (TermsAggregation.Entry termBucket : termBuckets) {
            termCounts.put(termBucket.getKeyAsString(), termBucket.getCount());
        }
        result.put(date, termCounts);
    }
    return ImmutableMap.copyOf(result);
}

Example 73 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project graylog2-server by Graylog2.

the class ESMessageList method doGenerateQueryPart.

@Override
public void doGenerateQueryPart(SearchJob job, Query query, MessageList messageList, ESGeneratedQueryContext queryContext) {
    final SearchSourceBuilder searchSourceBuilder = queryContext.searchSourceBuilder(messageList).size(messageList.limit()).from(messageList.offset());
    applyHighlightingIfActivated(searchSourceBuilder, job, query);
    final Set<String> effectiveStreamIds = messageList.effectiveStreams().isEmpty() ? query.usedStreamIds() : messageList.effectiveStreams();
    if (!messageList.fields().isEmpty()) {
        searchSourceBuilder.fetchSource(messageList.fields().toArray(new String[0]), new String[0]);
    }
    final List<Sort> sorts = firstNonNull(messageList.sort(), Collections.singletonList(Sort.create(Message.FIELD_TIMESTAMP, Sort.Order.DESC)));
    sorts.forEach(sort -> {
        final FieldSortBuilder fieldSort = SortBuilders.fieldSort(sort.field()).order(toSortOrder(sort.order()));
        final Optional<String> fieldType = queryContext.fieldType(effectiveStreamIds, sort.field());
        searchSourceBuilder.sort(fieldType.map(fieldSort::unmappedType).orElse(fieldSort));
    });
}

Example 74 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project graylog2-server by Graylog2.

the class ESPivot method doGenerateQueryPart.

@Override
public void doGenerateQueryPart(SearchJob job, Query query, Pivot pivot, ESGeneratedQueryContext queryContext) {
    LOG.debug("Generating aggregation for {}", pivot);
    final SearchSourceBuilder searchSourceBuilder = queryContext.searchSourceBuilder(pivot);
    final Map<Object, Object> contextMap = queryContext.contextMap();
    final AggTypes aggTypes = new AggTypes();
    contextMap.put(pivot.id(), aggTypes);
    // holds the initial level aggregation to be added to the query
    AggregationBuilder topLevelAggregation = null;
    // holds the last complete bucket aggregation into which subsequent buckets get added
    AggregationBuilder previousAggregation = null;
    // add global rollup series if those were requested
    if (pivot.rollup()) {
        seriesStream(pivot, queryContext, "global rollup").forEach(previousAggregation != null ? previousAggregation::subAggregation : searchSourceBuilder::aggregation);
    }
    final Iterator<BucketSpec> rowBuckets = pivot.rowGroups().iterator();
    while (rowBuckets.hasNext()) {
        final BucketSpec bucketSpec = rowBuckets.next();
        final String name = queryContext.nextName();
        LOG.debug("Creating row group aggregation '{}' as {}", bucketSpec.type(), name);
        final ESPivotBucketSpecHandler<? extends PivotSpec, ? extends Aggregation> handler = bucketHandlers.get(bucketSpec.type());
        if (handler == null) {
            throw new IllegalArgumentException("Unknown row_group type " + bucketSpec.type());
        }
        final Optional<AggregationBuilder> generatedAggregation = handler.createAggregation(name, pivot, bucketSpec, this, queryContext, query);
        if (generatedAggregation.isPresent()) {
            final AggregationBuilder aggregationBuilder = generatedAggregation.get();
            if (topLevelAggregation == null) {
                topLevelAggregation = aggregationBuilder;
            }
            // always insert the series for the final row group, or for each one if explicit rollup was requested
            if (!rowBuckets.hasNext() || pivot.rollup()) {
                seriesStream(pivot, queryContext, !rowBuckets.hasNext() ? "leaf row" : "row rollup").forEach(aggregationBuilder::subAggregation);
            }
            if (previousAggregation != null) {
                previousAggregation.subAggregation(aggregationBuilder);
            } else {
                searchSourceBuilder.aggregation(aggregationBuilder);
            }
            previousAggregation = aggregationBuilder;
        }
    }
    final Iterator<BucketSpec> colBuckets = pivot.columnGroups().iterator();
    while (colBuckets.hasNext()) {
        final BucketSpec bucketSpec = colBuckets.next();
        final String name = queryContext.nextName();
        LOG.debug("Creating column group aggregation '{}' as {}", bucketSpec.type(), name);
        final ESPivotBucketSpecHandler<? extends PivotSpec, ? extends Aggregation> handler = bucketHandlers.get(bucketSpec.type());
        if (handler == null) {
            throw new IllegalArgumentException("Unknown column_group type " + bucketSpec.type());
        }
        final Optional<AggregationBuilder> generatedAggregation = handler.createAggregation(name, pivot, bucketSpec, this, queryContext, query);
        if (generatedAggregation.isPresent()) {
            final AggregationBuilder aggregationBuilder = generatedAggregation.get();
            // always insert the series for the final row group, or for each one if explicit rollup was requested
            if (!colBuckets.hasNext() || pivot.rollup()) {
                seriesStream(pivot, queryContext, !colBuckets.hasNext() ? "leaf column" : "column rollup").forEach(aggregationBuilder::subAggregation);
            }
            if (previousAggregation != null) {
                previousAggregation.subAggregation(aggregationBuilder);
            } else {
                searchSourceBuilder.aggregation(aggregationBuilder);
            }
            previousAggregation = aggregationBuilder;
        }
    }
    final MinAggregationBuilder startTimestamp = AggregationBuilders.min("timestamp-min").field("timestamp");
    final MaxAggregationBuilder endTimestamp = AggregationBuilders.max("timestamp-max").field("timestamp");
    searchSourceBuilder.aggregation(startTimestamp);
    searchSourceBuilder.aggregation(endTimestamp);
    if (topLevelAggregation == null) {
        LOG.debug("No aggregations generated for {}", pivot);
    }
}

Example 75 with SearchSourceBuilder

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.builder.SearchSourceBuilder in project graylog2-server by Graylog2.

the class SearchesAdapterES6 method buildSearchRequest.

private SearchSourceBuilder buildSearchRequest(ScrollCommand scrollCommand) {
    final String query = normalizeQuery(scrollCommand.query());
    final QueryBuilder queryBuilder = isWildcardQuery(query) ? matchAllQuery() : queryStringQuery(query).allowLeadingWildcard(configuration.isAllowLeadingWildcardSearches());
    final Optional<BoolQueryBuilder> rangeQueryBuilder = scrollCommand.range().map(range -> QueryBuilders.boolQuery().must(TimeRangeQueryFactory.create(range)));
    final Optional<BoolQueryBuilder> filterQueryBuilder = scrollCommand.filter().filter(filter -> !isWildcardQuery(filter)).map(QueryBuilders::queryStringQuery).map(filter -> rangeQueryBuilder.orElse(QueryBuilders.boolQuery()).must(filter));
    final BoolQueryBuilder filteredQueryBuilder = QueryBuilders.boolQuery().must(queryBuilder);
    filterQueryBuilder.ifPresent(filteredQueryBuilder::filter);
    applyStreamsFilter(filteredQueryBuilder, scrollCommand);
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder().query(filteredQueryBuilder);
    applyPaginationIfPresent(searchSourceBuilder, scrollCommand);
    applySortingIfPresent(searchSourceBuilder, scrollCommand);
    applyHighlighting(searchSourceBuilder, scrollCommand);
    return searchSourceBuilder;
}