use of org.graylog.plugins.views.search.searchtypes.pivot.BucketSpec in project graylog2-server by Graylog2.
the class ESPivotTest method mixedPivotsAndSeriesShouldBeNested.
@Test
public void mixedPivotsAndSeriesShouldBeNested() {
final SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
final ESPivotBucketSpecHandler<? extends BucketSpec, ? extends Aggregation> valuesBucketHandler = mock(ESValuesHandler.class);
mockBucketSpecGeneratesComparableString(valuesBucketHandler);
final ESPivotBucketSpecHandler<? extends BucketSpec, ? extends Aggregation> timeBucketHandler = mock(ESTimeHandler.class);
mockBucketSpecGeneratesComparableString(timeBucketHandler);
final ESPivotSeriesSpecHandler<? extends SeriesSpec, ? extends Aggregation> countHandler = mock(ESCountHandler.class);
mockSeriesSpecGeneratesComparableString(countHandler);
bucketHandlers.put(Values.NAME, valuesBucketHandler);
bucketHandlers.put(Time.NAME, timeBucketHandler);
seriesHandlers.put(Count.NAME, countHandler);
when(queryContext.searchSourceBuilder(pivot)).thenReturn(searchSourceBuilder);
when(queryContext.nextName()).thenReturn("rowPivot1", "rowPivot2", "columnPivot1", "columnPivot2");
final BucketSpec rowPivot1 = Time.builder().field("timestamp").interval(AutoInterval.create()).build();
final BucketSpec rowPivot2 = Values.builder().field("http_method").limit(10).build();
final BucketSpec columnPivot1 = Values.builder().field("controller").limit(10).build();
final BucketSpec columnPivot2 = Values.builder().field("action").limit(10).build();
final Count count = Count.builder().build();
when(pivot.rowGroups()).thenReturn(ImmutableList.of(rowPivot1, rowPivot2));
when(pivot.columnGroups()).thenReturn(ImmutableList.of(columnPivot1, columnPivot2));
when(pivot.series()).thenReturn(Collections.singletonList(count));
when(pivot.rollup()).thenReturn(false);
when(queryContext.seriesName(any(), any())).thenCallRealMethod();
this.esPivot.doGenerateQueryPart(job, query, pivot, queryContext);
verify(timeBucketHandler).createAggregation(eq("rowPivot1"), eq(pivot), eq(rowPivot1), eq(this.esPivot), eq(queryContext), eq(query));
verify(valuesBucketHandler).createAggregation(eq("rowPivot2"), eq(pivot), eq(rowPivot2), eq(this.esPivot), eq(queryContext), eq(query));
verify(valuesBucketHandler).createAggregation(eq("columnPivot1"), eq(pivot), eq(columnPivot1), eq(this.esPivot), eq(queryContext), eq(query));
verify(valuesBucketHandler).createAggregation(eq("columnPivot2"), eq(pivot), eq(columnPivot2), eq(this.esPivot), eq(queryContext), eq(query));
final DocumentContext context = JsonPath.parse(searchSourceBuilder.toString());
extractAggregation(context, "rowPivot1").isEqualTo("Time{type=time, field=timestamp, interval=AutoInterval{type=auto, scaling=1.0}}");
extractAggregation(context, "rowPivot1.rowPivot2").isEqualTo("Values{type=values, field=http_method, limit=10}");
extractAggregation(context, "rowPivot1.rowPivot2.columnPivot1").isEqualTo("Values{type=values, field=controller, limit=10}");
extractAggregation(context, "rowPivot1.rowPivot2.columnPivot1.columnPivot2").isEqualTo("Values{type=values, field=action, limit=10}");
extractAggregation(context, "rowPivot1.rowPivot2.dummypivot-series-count()").isEqualTo("Count{type=count, id=count(), field=null}");
extractAggregation(context, "rowPivot1.rowPivot2.columnPivot1.columnPivot2.dummypivot-series-count()").isEqualTo("Count{type=count, id=count(), field=null}");
}
use of org.graylog.plugins.views.search.searchtypes.pivot.BucketSpec in project graylog2-server by Graylog2.
the class ESPivot method processRows.
/*
results from elasticsearch are nested so we need to recurse into the aggregation tree, but our result is a table, thus we need
to keep track of the current row keys manually
*/
private void processRows(PivotResult.Builder resultBuilder, SearchResponse searchResult, ESGeneratedQueryContext queryContext, Pivot pivot, List<BucketSpec> remainingRows, ArrayDeque<String> rowKeys, HasAggregations aggregation) {
if (remainingRows.isEmpty()) {
// this is the last row group, so we need to fork into the columns if they exist.
// being here also means that `rowKeys` contains the maximum number of parts, one for each combination of row bucket keys
// we will always add the series for this bucket, because that's the entire point of row groups
final PivotResult.Row.Builder rowBuilder = PivotResult.Row.builder().key(ImmutableList.copyOf(rowKeys));
// do the same for columns as we did for the rows
processColumns(rowBuilder, searchResult, queryContext, pivot, pivot.columnGroups(), new ArrayDeque<>(), aggregation);
// columnKeys is empty, because this is a rollup per row bucket, thus for all columns in that bucket (IOW it's not a leaf!)
if (pivot.rollup()) {
processSeries(rowBuilder, searchResult, queryContext, pivot, new ArrayDeque<>(), aggregation, true, "row-leaf");
}
resultBuilder.addRow(rowBuilder.source("leaf").build());
} else {
// this is not a leaf for the rows, so we add its key to the rowKeys and descend into the aggregation tree
// afterwards we'll check if we need to add rollup for intermediate buckets. not all clients need them so they can request
// to not calculate them
final BucketSpec currentBucket = remainingRows.get(0);
// this handler should never be missing, because we used it above to generate the query
// if it is missing for some weird reason, it's ok to fail hard here
final ESPivotBucketSpecHandler<? extends PivotSpec, ? extends Aggregation> handler = bucketHandlers.get(currentBucket.type());
final Aggregation aggregationResult = handler.extractAggregationFromResult(pivot, currentBucket, aggregation, queryContext);
final Stream<ESPivotBucketSpecHandler.Bucket> bucketStream = handler.handleResult(pivot, currentBucket, searchResult, aggregationResult, this, queryContext);
// for each bucket, recurse and eventually collect all the row keys. once we reach a leaf, we'll end up in the other if branch above
bucketStream.forEach(bucket -> {
// push the bucket's key and use its aggregation as the new source for sub-aggregations
rowKeys.addLast(bucket.key());
processRows(resultBuilder, searchResult, queryContext, pivot, tail(remainingRows), rowKeys, bucket.aggregation());
rowKeys.removeLast();
});
// also add the series for this row key if the client wants rollups
if (pivot.rollup()) {
final PivotResult.Row.Builder rowBuilder = PivotResult.Row.builder().key(ImmutableList.copyOf(rowKeys));
// columnKeys is empty, because this is a rollup per row bucket, thus for all columns in that bucket (IOW it's not a leaf!)
processSeries(rowBuilder, searchResult, queryContext, pivot, new ArrayDeque<>(), aggregation, true, "row-inner");
resultBuilder.addRow(rowBuilder.source("non-leaf").build());
}
}
}
use of org.graylog.plugins.views.search.searchtypes.pivot.BucketSpec in project graylog2-server by Graylog2.
the class ESPivot method processColumns.
private void processColumns(PivotResult.Row.Builder rowBuilder, SearchResponse searchResult, ESGeneratedQueryContext queryContext, Pivot pivot, List<BucketSpec> remainingColumns, ArrayDeque<String> columnKeys, HasAggregations aggregation) {
if (remainingColumns.isEmpty()) {
// with duplicate data entries
if (!columnKeys.isEmpty()) {
processSeries(rowBuilder, searchResult, queryContext, pivot, columnKeys, aggregation, false, "col-leaf");
}
} else {
// for a non-leaf column group, we need to recurse further into the aggregation tree
// and if rollup was requested we'll add intermediate series according to the column keys
final BucketSpec currentBucket = remainingColumns.get(0);
// this handler should never be missing, because we used it above to generate the query
// if it is missing for some weird reason, it's ok to fail hard here
final ESPivotBucketSpecHandler<? extends PivotSpec, ? extends Aggregation> handler = bucketHandlers.get(currentBucket.type());
final Aggregation aggregationResult = handler.extractAggregationFromResult(pivot, currentBucket, aggregation, queryContext);
final Stream<ESPivotBucketSpecHandler.Bucket> bucketStream = handler.handleResult(pivot, currentBucket, searchResult, aggregationResult, this, queryContext);
// for each bucket, recurse and eventually collect all the column keys. once we reach a leaf, we'll end up in the other if branch above
bucketStream.forEach(bucket -> {
// push the bucket's key and use its aggregation as the new source for sub-aggregations
columnKeys.addLast(bucket.key());
processColumns(rowBuilder, searchResult, queryContext, pivot, tail(remainingColumns), columnKeys, bucket.aggregation());
columnKeys.removeLast();
});
// don't add the empty column key rollup, because that's not the correct bucket here, it's being done in the row-leaf code
if (pivot.rollup() && !columnKeys.isEmpty()) {
// columnKeys is not empty, because this is a rollup per column in a row
processSeries(rowBuilder, searchResult, queryContext, pivot, columnKeys, aggregation, true, "col-inner");
}
}
}
use of org.graylog.plugins.views.search.searchtypes.pivot.BucketSpec in project graylog2-server by Graylog2.
the class ESPivot method doGenerateQueryPart.
@Override
public void doGenerateQueryPart(SearchJob job, Query query, Pivot pivot, ESGeneratedQueryContext queryContext) {
LOG.debug("Generating aggregation for {}", pivot);
final SearchSourceBuilder searchSourceBuilder = queryContext.searchSourceBuilder(pivot);
final Map<Object, Object> contextMap = queryContext.contextMap();
final AggTypes aggTypes = new AggTypes();
contextMap.put(pivot.id(), aggTypes);
// holds the initial level aggregation to be added to the query
AggregationBuilder topLevelAggregation = null;
// holds the last complete bucket aggregation into which subsequent buckets get added
AggregationBuilder previousAggregation = null;
// add global rollup series if those were requested
if (pivot.rollup()) {
seriesStream(pivot, queryContext, "global rollup").forEach(previousAggregation != null ? previousAggregation::subAggregation : searchSourceBuilder::aggregation);
}
final Iterator<BucketSpec> rowBuckets = pivot.rowGroups().iterator();
while (rowBuckets.hasNext()) {
final BucketSpec bucketSpec = rowBuckets.next();
final String name = queryContext.nextName();
LOG.debug("Creating row group aggregation '{}' as {}", bucketSpec.type(), name);
final ESPivotBucketSpecHandler<? extends PivotSpec, ? extends Aggregation> handler = bucketHandlers.get(bucketSpec.type());
if (handler == null) {
throw new IllegalArgumentException("Unknown row_group type " + bucketSpec.type());
}
final Optional<AggregationBuilder> generatedAggregation = handler.createAggregation(name, pivot, bucketSpec, this, queryContext, query);
if (generatedAggregation.isPresent()) {
final AggregationBuilder aggregationBuilder = generatedAggregation.get();
if (topLevelAggregation == null) {
topLevelAggregation = aggregationBuilder;
}
// always insert the series for the final row group, or for each one if explicit rollup was requested
if (!rowBuckets.hasNext() || pivot.rollup()) {
seriesStream(pivot, queryContext, !rowBuckets.hasNext() ? "leaf row" : "row rollup").forEach(aggregationBuilder::subAggregation);
}
if (previousAggregation != null) {
previousAggregation.subAggregation(aggregationBuilder);
} else {
searchSourceBuilder.aggregation(aggregationBuilder);
}
previousAggregation = aggregationBuilder;
}
}
final Iterator<BucketSpec> colBuckets = pivot.columnGroups().iterator();
while (colBuckets.hasNext()) {
final BucketSpec bucketSpec = colBuckets.next();
final String name = queryContext.nextName();
LOG.debug("Creating column group aggregation '{}' as {}", bucketSpec.type(), name);
final ESPivotBucketSpecHandler<? extends PivotSpec, ? extends Aggregation> handler = bucketHandlers.get(bucketSpec.type());
if (handler == null) {
throw new IllegalArgumentException("Unknown column_group type " + bucketSpec.type());
}
final Optional<AggregationBuilder> generatedAggregation = handler.createAggregation(name, pivot, bucketSpec, this, queryContext, query);
if (generatedAggregation.isPresent()) {
final AggregationBuilder aggregationBuilder = generatedAggregation.get();
// always insert the series for the final row group, or for each one if explicit rollup was requested
if (!colBuckets.hasNext() || pivot.rollup()) {
seriesStream(pivot, queryContext, !colBuckets.hasNext() ? "leaf column" : "column rollup").forEach(aggregationBuilder::subAggregation);
}
if (previousAggregation != null) {
previousAggregation.subAggregation(aggregationBuilder);
} else {
searchSourceBuilder.aggregation(aggregationBuilder);
}
previousAggregation = aggregationBuilder;
}
}
final MinAggregationBuilder startTimestamp = AggregationBuilders.min("timestamp-min").field("timestamp");
final MaxAggregationBuilder endTimestamp = AggregationBuilders.max("timestamp-max").field("timestamp");
searchSourceBuilder.aggregation(startTimestamp);
searchSourceBuilder.aggregation(endTimestamp);
if (topLevelAggregation == null) {
LOG.debug("No aggregations generated for {}", pivot);
}
}
use of org.graylog.plugins.views.search.searchtypes.pivot.BucketSpec in project graylog2-server by Graylog2.
the class PivotAggregationSearch method getAggregationQuery.
/**
* Returns the query to compute the aggregation.
*
* @param parameters processor parameters
* @param searchWithinMs processor search within period. Used to build the date range buckets
* @param executeEveryMs
* @return aggregation query
*/
private Query getAggregationQuery(AggregationEventProcessorParameters parameters, long searchWithinMs, long executeEveryMs) {
final Pivot.Builder pivotBuilder = Pivot.builder().id(PIVOT_ID).rollup(true);
final ImmutableList<SeriesSpec> series = config.series().stream().map(entry -> entry.function().toSeriesSpec(metricName(entry), entry.field().orElse(null))).collect(ImmutableList.toImmutableList());
if (!series.isEmpty()) {
pivotBuilder.series(series);
}
// Wrap every aggregation with date range buckets of the searchWithin time range.
// If the aggregation is configured to be using a sliding window (searchWithin > executeEveryMs)
// the time ranges will overlap.
// This allows us to run aggregations over larger time ranges than the searchWithin time.
// The results will be received in time buckets of the searchWithin time size.
final DateRangeBucket dateRangeBucket = buildDateRangeBuckets(parameters.timerange(), searchWithinMs, executeEveryMs);
final List<BucketSpec> groupBy = new ArrayList<>();
// The first bucket must be the date range!
groupBy.add(dateRangeBucket);
if (!config.groupBy().isEmpty()) {
// Then we add the configured groups
groupBy.addAll(config.groupBy().stream().map(field -> Values.builder().limit(Integer.MAX_VALUE).field(field).build()).collect(Collectors.toList()));
}
// We always have row groups because of the date range buckets
pivotBuilder.rowGroups(groupBy);
final Set<SearchType> searchTypes = Collections.singleton(pivotBuilder.build());
final Query.Builder queryBuilder = Query.builder().id(QUERY_ID).searchTypes(searchTypes).query(ElasticsearchQueryString.of(config.query())).timerange(parameters.timerange());
final Set<String> streams = getStreams(parameters);
if (!streams.isEmpty()) {
queryBuilder.filter(filteringForStreamIds(streams));
}
return queryBuilder.build();
}
Aggregations