Examples with Aggregations org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregations used on opensource projects

Example 31 with Aggregations

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregations in project graylog2-server by Graylog2.

the class ESMessageList method doExtractResult.

@Override
public SearchType.Result doExtractResult(SearchJob job, Query query, MessageList searchType, org.graylog.shaded.elasticsearch7.org.elasticsearch.action.search.SearchResponse result, Aggregations aggregations, ESGeneratedQueryContext queryContext) {
    final List<ResultMessageSummary> messages = StreamSupport.stream(result.getHits().spliterator(), false).map(ESMessageList::resultMessageFromSearchHit).map((resultMessage) -> ResultMessageSummary.create(resultMessage.highlightRanges, resultMessage.getMessage().getFields(), resultMessage.getIndex())).collect(Collectors.toList());
    final String undecoratedQueryString = query.query().queryString();
    final String queryString = this.esQueryDecorators.decorate(undecoratedQueryString, job, query);
    final DateTime from = query.effectiveTimeRange(searchType).getFrom();
    final DateTime to = query.effectiveTimeRange(searchType).getTo();
    final SearchResponse searchResponse = SearchResponse.create(undecoratedQueryString, queryString, Collections.emptySet(), messages, Collections.emptySet(), 0, result.getHits().getTotalHits().value, from, to);
    final SearchResponse decoratedSearchResponse = decoratorProcessor.decorateSearchResponse(searchResponse, searchType.decorators());
    final MessageList.Result.Builder resultBuilder = MessageList.Result.result(searchType.id()).messages(decoratedSearchResponse.messages()).effectiveTimerange(AbsoluteRange.create(from, to)).totalResults(decoratedSearchResponse.totalResults());
    return searchType.name().map(resultBuilder::name).orElse(resultBuilder).build();
}

Example 32 with Aggregations

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregations in project graylog2-server by Graylog2.

the class ESPivot method doGenerateQueryPart.

@Override
public void doGenerateQueryPart(SearchJob job, Query query, Pivot pivot, ESGeneratedQueryContext queryContext) {
    LOG.debug("Generating aggregation for {}", pivot);
    final SearchSourceBuilder searchSourceBuilder = queryContext.searchSourceBuilder(pivot);
    final Map<Object, Object> contextMap = queryContext.contextMap();
    final AggTypes aggTypes = new AggTypes();
    contextMap.put(pivot.id(), aggTypes);
    // holds the initial level aggregation to be added to the query
    AggregationBuilder topLevelAggregation = null;
    // holds the last complete bucket aggregation into which subsequent buckets get added
    AggregationBuilder previousAggregation = null;
    // add global rollup series if those were requested
    if (pivot.rollup()) {
        seriesStream(pivot, queryContext, "global rollup").forEach(searchSourceBuilder::aggregation);
    }
    final Iterator<BucketSpec> rowBuckets = pivot.rowGroups().iterator();
    while (rowBuckets.hasNext()) {
        final BucketSpec bucketSpec = rowBuckets.next();
        final String name = queryContext.nextName();
        LOG.debug("Creating row group aggregation '{}' as {}", bucketSpec.type(), name);
        final ESPivotBucketSpecHandler<? extends PivotSpec, ? extends Aggregation> handler = bucketHandlers.get(bucketSpec.type());
        if (handler == null) {
            throw new IllegalArgumentException("Unknown row_group type " + bucketSpec.type());
        }
        final Optional<AggregationBuilder> generatedAggregation = handler.createAggregation(name, pivot, bucketSpec, this, queryContext, query);
        if (generatedAggregation.isPresent()) {
            final AggregationBuilder aggregationBuilder = generatedAggregation.get();
            if (topLevelAggregation == null) {
                topLevelAggregation = aggregationBuilder;
            }
            // always insert the series for the final row group, or for each one if explicit rollup was requested
            if (!rowBuckets.hasNext() || pivot.rollup()) {
                seriesStream(pivot, queryContext, !rowBuckets.hasNext() ? "leaf row" : "row rollup").forEach(aggregationBuilder::subAggregation);
            }
            if (previousAggregation != null) {
                previousAggregation.subAggregation(aggregationBuilder);
            } else {
                searchSourceBuilder.aggregation(aggregationBuilder);
            }
            previousAggregation = aggregationBuilder;
        }
    }
    final Iterator<BucketSpec> colBuckets = pivot.columnGroups().iterator();
    while (colBuckets.hasNext()) {
        final BucketSpec bucketSpec = colBuckets.next();
        final String name = queryContext.nextName();
        LOG.debug("Creating column group aggregation '{}' as {}", bucketSpec.type(), name);
        final ESPivotBucketSpecHandler<? extends PivotSpec, ? extends Aggregation> handler = bucketHandlers.get(bucketSpec.type());
        if (handler == null) {
            throw new IllegalArgumentException("Unknown column_group type " + bucketSpec.type());
        }
        final Optional<AggregationBuilder> generatedAggregation = handler.createAggregation(name, pivot, bucketSpec, this, queryContext, query);
        if (generatedAggregation.isPresent()) {
            final AggregationBuilder aggregationBuilder = generatedAggregation.get();
            // always insert the series for the final row group, or for each one if explicit rollup was requested
            if (!colBuckets.hasNext() || pivot.rollup()) {
                seriesStream(pivot, queryContext, !colBuckets.hasNext() ? "leaf column" : "column rollup").forEach(aggregationBuilder::subAggregation);
            }
            if (previousAggregation != null) {
                previousAggregation.subAggregation(aggregationBuilder);
            } else {
                searchSourceBuilder.aggregation(aggregationBuilder);
            }
            previousAggregation = aggregationBuilder;
        }
    }
    final MinAggregationBuilder startTimestamp = AggregationBuilders.min("timestamp-min").field("timestamp");
    final MaxAggregationBuilder endTimestamp = AggregationBuilders.max("timestamp-max").field("timestamp");
    searchSourceBuilder.aggregation(startTimestamp);
    searchSourceBuilder.aggregation(endTimestamp);
    if (topLevelAggregation == null) {
        LOG.debug("No aggregations generated for {}", pivot);
    }
}

Example 33 with Aggregations

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregations in project graylog2-server by Graylog2.

the class ESPivot method doExtractResult.

@Override
public SearchType.Result doExtractResult(SearchJob job, Query query, Pivot pivot, SearchResponse queryResult, Aggregations aggregations, ESGeneratedQueryContext queryContext) {
    final AbsoluteRange effectiveTimerange = extractEffectiveTimeRange(queryResult, query, pivot);
    final PivotResult.Builder resultBuilder = PivotResult.builder().id(pivot.id()).effectiveTimerange(effectiveTimerange).total(extractDocumentCount(queryResult, pivot, queryContext));
    // pivot results are a table where cells can contain multiple "values" and not only scalars:
    // each combination of row and column groups can contain all series (if rollup is true)
    // if rollup is false, only the "leaf" components contain the series
    // in the elasticsearch result, rows and columns are simply nested aggregations (first aggregations from rows, then from columns)
    // with metric aggregations on the corresponding levels.
    // first we iterate over all row groups (whose values generate a "key array", corresponding to the nesting level)
    // once we exhaust the row groups, we descend into the columns, which get added as values to their corresponding rows
    // on each nesting level and combination we have to check for series which we also add as values to the containing row
    final HasAggregations initialResult = createInitialResult(queryResult);
    processRows(resultBuilder, queryResult, queryContext, pivot, pivot.rowGroups(), new ArrayDeque<>(), initialResult);
    return pivot.name().map(resultBuilder::name).orElse(resultBuilder).build();
}

Example 34 with Aggregations

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregations in project graylog2-server by Graylog2.

the class ElasticsearchBackend method doRun.

@Override
public QueryResult doRun(SearchJob job, Query query, ESGeneratedQueryContext queryContext) {
    if (query.searchTypes().isEmpty()) {
        return QueryResult.builder().query(query).searchTypes(Collections.emptyMap()).errors(new HashSet<>(queryContext.errors())).build();
    }
    LOG.debug("Running query {} for job {}", query.id(), job.getId());
    final HashMap<String, SearchType.Result> resultsMap = Maps.newHashMap();
    final Set<String> affectedIndices = indexLookup.indexNamesForStreamsInTimeRange(query.usedStreamIds(), query.timerange());
    final Map<String, SearchSourceBuilder> searchTypeQueries = queryContext.searchTypeQueries();
    final List<String> searchTypeIds = new ArrayList<>(searchTypeQueries.keySet());
    final List<SearchRequest> searches = searchTypeIds.stream().map(searchTypeId -> {
        final Set<String> affectedIndicesForSearchType = query.searchTypes().stream().filter(s -> s.id().equalsIgnoreCase(searchTypeId)).findFirst().flatMap(searchType -> {
            if (searchType.effectiveStreams().isEmpty() && !query.globalOverride().flatMap(GlobalOverride::timerange).isPresent() && !searchType.timerange().isPresent()) {
                return Optional.empty();
            }
            final Set<String> usedStreamIds = searchType.effectiveStreams().isEmpty() ? query.usedStreamIds() : searchType.effectiveStreams();
            return Optional.of(indexLookup.indexNamesForStreamsInTimeRange(usedStreamIds, query.effectiveTimeRange(searchType)));
        }).orElse(affectedIndices);
        Set<String> indices = affectedIndicesForSearchType.isEmpty() ? Collections.singleton("") : affectedIndicesForSearchType;
        return new SearchRequest().source(searchTypeQueries.get(searchTypeId)).indices(indices.toArray(new String[0])).indicesOptions(IndicesOptions.fromOptions(false, false, true, false));
    }).collect(Collectors.toList());
    final List<MultiSearchResponse.Item> results = client.msearch(searches, "Unable to perform search query: ");
    for (SearchType searchType : query.searchTypes()) {
        final String searchTypeId = searchType.id();
        final Provider<ESSearchTypeHandler<? extends SearchType>> handlerProvider = elasticsearchSearchTypeHandlers.get(searchType.type());
        if (handlerProvider == null) {
            LOG.error("Unknown search type '{}', cannot convert query result.", searchType.type());
            // no need to add another error here, as the query generation code will have added the error about the missing handler already
            continue;
        }
        if (isSearchTypeWithError(queryContext, searchTypeId)) {
            LOG.error("Failed search type '{}', cannot convert query result, skipping.", searchType.type());
            // no need to add another error here, as the query generation code will have added the error about the missing handler already
            continue;
        }
        // we create a new instance because some search type handlers might need to track information between generating the query and
        // processing its result, such as aggregations, which depend on the name and type
        final ESSearchTypeHandler<? extends SearchType> handler = handlerProvider.get();
        final int searchTypeIndex = searchTypeIds.indexOf(searchTypeId);
        final MultiSearchResponse.Item multiSearchResponse = results.get(searchTypeIndex);
        if (multiSearchResponse.isFailure()) {
            ElasticsearchException e = new ElasticsearchException("Search type returned error: ", multiSearchResponse.getFailure());
            queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
        } else if (checkForFailedShards(multiSearchResponse).isPresent()) {
            ElasticsearchException e = checkForFailedShards(multiSearchResponse).get();
            queryContext.addError(SearchTypeErrorParser.parse(query, searchTypeId, e));
        } else {
            final SearchType.Result searchTypeResult = handler.extractResult(job, query, searchType, multiSearchResponse.getResponse(), queryContext);
            if (searchTypeResult != null) {
                resultsMap.put(searchTypeId, searchTypeResult);
            }
        }
    }
    LOG.debug("Query {} ran for job {}", query.id(), job.getId());
    return QueryResult.builder().query(query).searchTypes(resultsMap).errors(new HashSet<>(queryContext.errors())).build();
}

Example 35 with Aggregations

use of org.graylog.shaded.elasticsearch7.org.elasticsearch.search.aggregations.Aggregations in project graylog2-server by Graylog2.

the class ESEventListTest method testSortingOfStreamsInDoExtractResult.

@Test
public void testSortingOfStreamsInDoExtractResult() {
    final ESEventList esEventList = new TestESEventList();
    final SearchJob searchJob = mock(SearchJob.class);
    final Query query = mock(Query.class);
    final SearchResponse searchResult = mock(SearchResponse.class);
    final Aggregations metricAggregation = mock(Aggregations.class);
    final ESGeneratedQueryContext queryContext = mock(ESGeneratedQueryContext.class);
    final EventList eventList = EventList.builder().id("search-type-id").streams(ImmutableSet.of("stream-id-1", "stream-id-2")).build();
    final EventList.Result eventResult = (EventList.Result) esEventList.doExtractResult(searchJob, query, eventList, searchResult, metricAggregation, queryContext);
    assertThat(eventResult.events()).containsExactly(eventSummary("find-1", ImmutableSet.of("stream-id-1")), eventSummary("find-2", ImmutableSet.of("stream-id-2")), eventSummary("find-3", ImmutableSet.of("stream-id-1", "stream-id-2")));
}