Examples with Aggregation org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregation used on opensource projects

Example 26 with Aggregation

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregation in project graylog2-server by Graylog2.

the class ESPivotTest method mixedPivotsAndSeriesShouldBeNested.

@Test
public void mixedPivotsAndSeriesShouldBeNested() {
    final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
    final ESPivotBucketSpecHandler<? extends BucketSpec, ? extends Aggregation> valuesBucketHandler = mock(ESValuesHandler.class);
    mockBucketSpecGeneratesComparableString(valuesBucketHandler);
    final ESPivotBucketSpecHandler<? extends BucketSpec, ? extends Aggregation> timeBucketHandler = mock(ESTimeHandler.class);
    mockBucketSpecGeneratesComparableString(timeBucketHandler);
    final ESPivotSeriesSpecHandler<? extends SeriesSpec, ? extends Aggregation> countHandler = mock(ESCountHandler.class);
    mockSeriesSpecGeneratesComparableString(countHandler);
    bucketHandlers.put(Values.NAME, valuesBucketHandler);
    bucketHandlers.put(Time.NAME, timeBucketHandler);
    seriesHandlers.put(Count.NAME, countHandler);
    when(queryContext.searchSourceBuilder(pivot)).thenReturn(searchSourceBuilder);
    when(queryContext.nextName()).thenReturn("rowPivot1", "rowPivot2", "columnPivot1", "columnPivot2");
    final BucketSpec rowPivot1 = Time.builder().field("timestamp").interval(AutoInterval.create()).build();
    final BucketSpec rowPivot2 = Values.builder().field("http_method").limit(10).build();
    final BucketSpec columnPivot1 = Values.builder().field("controller").limit(10).build();
    final BucketSpec columnPivot2 = Values.builder().field("action").limit(10).build();
    final Count count = Count.builder().build();
    when(pivot.rowGroups()).thenReturn(ImmutableList.of(rowPivot1, rowPivot2));
    when(pivot.columnGroups()).thenReturn(ImmutableList.of(columnPivot1, columnPivot2));
    when(pivot.series()).thenReturn(Collections.singletonList(count));
    when(pivot.rollup()).thenReturn(false);
    when(queryContext.seriesName(any(), any())).thenCallRealMethod();
    this.esPivot.doGenerateQueryPart(job, query, pivot, queryContext);
    verify(timeBucketHandler).createAggregation(eq("rowPivot1"), eq(pivot), eq(rowPivot1), eq(this.esPivot), eq(queryContext), eq(query));
    verify(valuesBucketHandler).createAggregation(eq("rowPivot2"), eq(pivot), eq(rowPivot2), eq(this.esPivot), eq(queryContext), eq(query));
    verify(valuesBucketHandler).createAggregation(eq("columnPivot1"), eq(pivot), eq(columnPivot1), eq(this.esPivot), eq(queryContext), eq(query));
    verify(valuesBucketHandler).createAggregation(eq("columnPivot2"), eq(pivot), eq(columnPivot2), eq(this.esPivot), eq(queryContext), eq(query));
    final DocumentContext context = JsonPath.parse(searchSourceBuilder.toString());
    extractAggregation(context, "rowPivot1").isEqualTo("Time{type=time, field=timestamp, interval=AutoInterval{type=auto, scaling=1.0}}");
    extractAggregation(context, "rowPivot1.rowPivot2").isEqualTo("Values{type=values, field=http_method, limit=10}");
    extractAggregation(context, "rowPivot1.rowPivot2.columnPivot1").isEqualTo("Values{type=values, field=controller, limit=10}");
    extractAggregation(context, "rowPivot1.rowPivot2.columnPivot1.columnPivot2").isEqualTo("Values{type=values, field=action, limit=10}");
    extractAggregation(context, "rowPivot1.rowPivot2.dummypivot-series-count()").isEqualTo("Count{type=count, id=count(), field=null}");
    extractAggregation(context, "rowPivot1.rowPivot2.columnPivot1.columnPivot2.dummypivot-series-count()").isEqualTo("Count{type=count, id=count(), field=null}");
}

Example 27 with Aggregation

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregation in project graylog2-server by Graylog2.

the class QuerySuggestionsES7 method suggest.

@Override
public SuggestionResponse suggest(SuggestionRequest req) {
    final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(req.streams(), req.timerange());
    final TermSuggestionBuilder suggestionBuilder = SuggestBuilders.termSuggestion(req.field()).text(req.input()).size(req.size());
    final SearchSourceBuilder search = new SearchSourceBuilder().query(QueryBuilders.prefixQuery(req.field(), req.input())).size(0).aggregation(AggregationBuilders.terms("fieldvalues").field(req.field()).size(req.size())).suggest(new SuggestBuilder().addSuggestion("corrections", suggestionBuilder));
    try {
        final SearchResponse result = client.singleSearch(new SearchRequest(affectedIndices.toArray(new String[] {})).source(search), "Failed to execute aggregation");
        final ParsedStringTerms fieldValues = result.getAggregations().get("fieldvalues");
        final List<SuggestionEntry> entries = fieldValues.getBuckets().stream().map(b -> new SuggestionEntry(b.getKeyAsString(), b.getDocCount())).collect(Collectors.toList());
        if (!entries.isEmpty()) {
            return SuggestionResponse.forSuggestions(req.field(), req.input(), entries, fieldValues.getSumOfOtherDocCounts());
        } else {
            TermSuggestion suggestion = result.getSuggest().getSuggestion("corrections");
            final List<SuggestionEntry> corrections = suggestion.getEntries().stream().flatMap(e -> e.getOptions().stream()).map(o -> new SuggestionEntry(o.getText().string(), o.getFreq())).collect(Collectors.toList());
            return SuggestionResponse.forSuggestions(req.field(), req.input(), corrections, null);
        }
    } catch (org.graylog.shaded.elasticsearch7.org.elasticsearch.ElasticsearchException exception) {
        final SuggestionError err = tryResponseException(exception).orElseGet(() -> parseException(exception));
        return SuggestionResponse.forError(req.field(), req.input(), err);
    }
}

Example 28 with Aggregation

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregation in project graylog2-server by Graylog2.

the class ESPivot method processRows.

/*
        results from elasticsearch are nested so we need to recurse into the aggregation tree, but our result is a table, thus we need
        to keep track of the current row keys manually
         */
private void processRows(PivotResult.Builder resultBuilder, SearchResponse searchResult, ESGeneratedQueryContext queryContext, Pivot pivot, List<BucketSpec> remainingRows, ArrayDeque<String> rowKeys, HasAggregations aggregation) {
    if (remainingRows.isEmpty()) {
        // this is the last row group, so we need to fork into the columns if they exist.
        // being here also means that `rowKeys` contains the maximum number of parts, one for each combination of row bucket keys
        // we will always add the series for this bucket, because that's the entire point of row groups
        final PivotResult.Row.Builder rowBuilder = PivotResult.Row.builder().key(ImmutableList.copyOf(rowKeys));
        // do the same for columns as we did for the rows
        processColumns(rowBuilder, searchResult, queryContext, pivot, pivot.columnGroups(), new ArrayDeque<>(), aggregation);
        // columnKeys is empty, because this is a rollup per row bucket, thus for all columns in that bucket (IOW it's not a leaf!)
        if (pivot.rollup()) {
            processSeries(rowBuilder, searchResult, queryContext, pivot, new ArrayDeque<>(), aggregation, true, "row-leaf");
        }
        resultBuilder.addRow(rowBuilder.source("leaf").build());
    } else {
        // this is not a leaf for the rows, so we add its key to the rowKeys and descend into the aggregation tree
        // afterwards we'll check if we need to add rollup for intermediate buckets. not all clients need them so they can request
        // to not calculate them
        final BucketSpec currentBucket = remainingRows.get(0);
        // this handler should never be missing, because we used it above to generate the query
        // if it is missing for some weird reason, it's ok to fail hard here
        final ESPivotBucketSpecHandler<? extends PivotSpec, ? extends Aggregation> handler = bucketHandlers.get(currentBucket.type());
        final Aggregation aggregationResult = handler.extractAggregationFromResult(pivot, currentBucket, aggregation, queryContext);
        final Stream<ESPivotBucketSpecHandler.Bucket> bucketStream = handler.handleResult(pivot, currentBucket, searchResult, aggregationResult, this, queryContext);
        // for each bucket, recurse and eventually collect all the row keys. once we reach a leaf, we'll end up in the other if branch above
        bucketStream.forEach(bucket -> {
            // push the bucket's key and use its aggregation as the new source for sub-aggregations
            rowKeys.addLast(bucket.key());
            processRows(resultBuilder, searchResult, queryContext, pivot, tail(remainingRows), rowKeys, bucket.aggregation());
            rowKeys.removeLast();
        });
        // also add the series for this row key if the client wants rollups
        if (pivot.rollup()) {
            final PivotResult.Row.Builder rowBuilder = PivotResult.Row.builder().key(ImmutableList.copyOf(rowKeys));
            // columnKeys is empty, because this is a rollup per row bucket, thus for all columns in that bucket (IOW it's not a leaf!)
            processSeries(rowBuilder, searchResult, queryContext, pivot, new ArrayDeque<>(), aggregation, true, "row-inner");
            resultBuilder.addRow(rowBuilder.source("non-leaf").build());
        }
    }
}

Example 29 with Aggregation

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregation in project graylog2-server by Graylog2.

the class ESPivot method processColumns.

private void processColumns(PivotResult.Row.Builder rowBuilder, SearchResponse searchResult, ESGeneratedQueryContext queryContext, Pivot pivot, List<BucketSpec> remainingColumns, ArrayDeque<String> columnKeys, HasAggregations aggregation) {
    if (remainingColumns.isEmpty()) {
        // with duplicate data entries
        if (!columnKeys.isEmpty()) {
            processSeries(rowBuilder, searchResult, queryContext, pivot, columnKeys, aggregation, false, "col-leaf");
        }
    } else {
        // for a non-leaf column group, we need to recurse further into the aggregation tree
        // and if rollup was requested we'll add intermediate series according to the column keys
        final BucketSpec currentBucket = remainingColumns.get(0);
        // this handler should never be missing, because we used it above to generate the query
        // if it is missing for some weird reason, it's ok to fail hard here
        final ESPivotBucketSpecHandler<? extends PivotSpec, ? extends Aggregation> handler = bucketHandlers.get(currentBucket.type());
        final Aggregation aggregationResult = handler.extractAggregationFromResult(pivot, currentBucket, aggregation, queryContext);
        final Stream<ESPivotBucketSpecHandler.Bucket> bucketStream = handler.handleResult(pivot, currentBucket, searchResult, aggregationResult, this, queryContext);
        // for each bucket, recurse and eventually collect all the column keys. once we reach a leaf, we'll end up in the other if branch above
        bucketStream.forEach(bucket -> {
            // push the bucket's key and use its aggregation as the new source for sub-aggregations
            columnKeys.addLast(bucket.key());
            processColumns(rowBuilder, searchResult, queryContext, pivot, tail(remainingColumns), columnKeys, bucket.aggregation());
            columnKeys.removeLast();
        });
        // don't add the empty column key rollup, because that's not the correct bucket here, it's being done in the row-leaf code
        if (pivot.rollup() && !columnKeys.isEmpty()) {
            // columnKeys is not empty, because this is a rollup per column in a row
            processSeries(rowBuilder, searchResult, queryContext, pivot, columnKeys, aggregation, true, "col-inner");
        }
    }
}

Example 30 with Aggregation

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregation in project graylog2-server by Graylog2.

the class ESCountHandler method doCreateAggregation.

@Nonnull
@Override
public Optional<AggregationBuilder> doCreateAggregation(String name, Pivot pivot, Count count, ESPivot searchTypeHandler, ESGeneratedQueryContext queryContext) {
    final String field = count.field();
    if (field == null) {
        // doc_count is always present in elasticsearch's bucket aggregations, no need to add it
        return Optional.empty();
    } else {
        // the request was for a field count, we have to add a value_count sub aggregation
        final ValueCountAggregationBuilder value = AggregationBuilders.count(name).field(field);
        record(queryContext, pivot, count, name, ValueCount.class);
        return Optional.of(value);
    }
}