Search in sources :

Example 11 with MetricAggregation

use of io.searchbox.core.search.aggregation.MetricAggregation in project graylog2-server by Graylog2.

the class ESPivot method processSeries.

private void processSeries(PivotResult.Row.Builder rowBuilder, SearchResult searchResult, ESGeneratedQueryContext queryContext, Pivot pivot, ArrayDeque<String> columnKeys, MetricAggregation aggregation, boolean rollup, String source) {
    pivot.series().forEach(seriesSpec -> {
        final ESPivotSeriesSpecHandler<? extends SeriesSpec, ? extends Aggregation> seriesHandler = seriesHandlers.get(seriesSpec.type());
        final Aggregation series = seriesHandler.extractAggregationFromResult(pivot, seriesSpec, aggregation, queryContext);
        seriesHandler.handleResult(pivot, seriesSpec, searchResult, series, this, queryContext).map(value -> {
            columnKeys.addLast(value.id());
            final PivotResult.Value v = PivotResult.Value.create(columnKeys, value.value(), rollup, source);
            columnKeys.removeLast();
            return v;
        }).forEach(rowBuilder::addValue);
    });
}
Also used : Aggregation(io.searchbox.core.search.aggregation.Aggregation) MetricAggregation(io.searchbox.core.search.aggregation.MetricAggregation) ESGeneratedQueryContext(org.graylog.storage.elasticsearch6.views.ESGeneratedQueryContext) ESSearchTypeHandler(org.graylog.storage.elasticsearch6.views.searchtypes.ESSearchTypeHandler) MinAggregationBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.aggregations.metrics.min.MinAggregationBuilder) DateTimeZone(org.joda.time.DateTimeZone) InvalidRangeParametersException(org.graylog2.plugin.indexer.searches.timeranges.InvalidRangeParametersException) Query(org.graylog.plugins.views.search.Query) PivotResult(org.graylog.plugins.views.search.searchtypes.pivot.PivotResult) LoggerFactory(org.slf4j.LoggerFactory) RelativeRange(org.graylog2.plugin.indexer.searches.timeranges.RelativeRange) Inject(javax.inject.Inject) Tuple2(org.jooq.lambda.tuple.Tuple2) ImmutableList(com.google.common.collect.ImmutableList) SearchType(org.graylog.plugins.views.search.SearchType) BucketSpec(org.graylog.plugins.views.search.searchtypes.pivot.BucketSpec) SeriesSpec(org.graylog.plugins.views.search.searchtypes.pivot.SeriesSpec) Map(java.util.Map) AbsoluteRange(org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange) MaxAggregationBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.aggregations.metrics.max.MaxAggregationBuilder) AggregationBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.aggregations.AggregationBuilder) Pivot(org.graylog.plugins.views.search.searchtypes.pivot.Pivot) TimeRange(org.graylog2.plugin.indexer.searches.timeranges.TimeRange) PivotSpec(org.graylog.plugins.views.search.searchtypes.pivot.PivotSpec) SearchJob(org.graylog.plugins.views.search.SearchJob) Logger(org.slf4j.Logger) IdentityHashMap(java.util.IdentityHashMap) Iterator(java.util.Iterator) DateTime(org.joda.time.DateTime) Aggregation(io.searchbox.core.search.aggregation.Aggregation) EntryStream(one.util.streamex.EntryStream) List(java.util.List) Tuple(org.jooq.lambda.tuple.Tuple) Stream(java.util.stream.Stream) AggregationBuilders(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.aggregations.AggregationBuilders) MetricAggregation(io.searchbox.core.search.aggregation.MetricAggregation) Optional(java.util.Optional) Preconditions(com.google.common.base.Preconditions) ArrayDeque(java.util.ArrayDeque) SearchSourceBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.builder.SearchSourceBuilder) SearchResult(io.searchbox.core.SearchResult)

Example 12 with MetricAggregation

use of io.searchbox.core.search.aggregation.MetricAggregation in project herd by FINRAOS.

the class ElasticSearchHelperTest method testGetResultTypeIndexSearchResponseDtoSearchResult.

@Test
public void testGetResultTypeIndexSearchResponseDtoSearchResult() {
    SearchResult searchResult = mock(SearchResult.class);
    MetricAggregation metricAggregation = mock(MetricAggregation.class);
    TermsAggregation termsAggregation = mock(TermsAggregation.class);
    List<TermsAggregation.Entry> buckets = new ArrayList<>();
    buckets.add(new TermsAggregation("TermAggregation", new JsonObject()).new Entry(new JsonObject(), "key", 1L));
    when(searchResult.getAggregations()).thenReturn(metricAggregation);
    when(metricAggregation.getTermsAggregation(RESULT_TYPE_AGGS)).thenReturn(termsAggregation);
    when(termsAggregation.getBuckets()).thenReturn(buckets);
    List<ResultTypeIndexSearchResponseDto> result = elasticsearchHelper.getResultTypeIndexSearchResponseDto(searchResult);
    assertThat("Result is null.", result, is(notNullValue()));
}
Also used : TermsAggregation(io.searchbox.core.search.aggregation.TermsAggregation) MetricAggregation(io.searchbox.core.search.aggregation.MetricAggregation) ArrayList(java.util.ArrayList) JsonObject(com.google.gson.JsonObject) ResultTypeIndexSearchResponseDto(org.finra.herd.model.dto.ResultTypeIndexSearchResponseDto) SearchResult(io.searchbox.core.SearchResult) Test(org.junit.Test) AbstractDaoTest(org.finra.herd.dao.AbstractDaoTest)

Example 13 with MetricAggregation

use of io.searchbox.core.search.aggregation.MetricAggregation in project herd by FINRAOS.

the class ElasticsearchHelper method getNestedTagTagIndexSearchResponseDto.

/**
 * create tag tag index response dto
 *
 * @param searchResult search result
 *
 * @return tag type index search response dto list
 */
public List<TagTypeIndexSearchResponseDto> getNestedTagTagIndexSearchResponseDto(SearchResult searchResult) {
    MetricAggregation metricAggregation = searchResult.getAggregations();
    MetricAggregation tagFacetAggregation = metricAggregation.getSumAggregation(TAG_FACET_AGGS);
    TermsAggregation tagTypeCodesAggregation = tagFacetAggregation.getTermsAggregation(TAGTYPE_CODE_AGGREGATION);
    return getTagTypeIndexSearchResponseDtosFromTermsAggregation(tagTypeCodesAggregation);
}
Also used : TermsAggregation(io.searchbox.core.search.aggregation.TermsAggregation) MetricAggregation(io.searchbox.core.search.aggregation.MetricAggregation)

Example 14 with MetricAggregation

use of io.searchbox.core.search.aggregation.MetricAggregation in project graylog2-server by Graylog2.

the class ESMessageList method doExtractResult.

@Override
public SearchType.Result doExtractResult(SearchJob job, Query query, MessageList searchType, SearchResult result, MetricAggregation aggregations, ESGeneratedQueryContext queryContext) {
    // noinspection unchecked
    final List<ResultMessageSummary> messages = result.getHits(Map.class, false).stream().map(hit -> ResultMessage.parseFromSource(hit.id, hit.index, (Map<String, Object>) hit.source, hit.highlight)).map((resultMessage) -> ResultMessageSummary.create(resultMessage.highlightRanges, resultMessage.getMessage().getFields(), resultMessage.getIndex())).collect(Collectors.toList());
    final String undecoratedQueryString = query.query().queryString();
    final String queryString = this.esQueryDecorators.decorate(undecoratedQueryString, job, query);
    final DateTime from = query.effectiveTimeRange(searchType).getFrom();
    final DateTime to = query.effectiveTimeRange(searchType).getTo();
    final SearchResponse searchResponse = SearchResponse.create(undecoratedQueryString, queryString, Collections.emptySet(), messages, Collections.emptySet(), 0, result.getTotal(), from, to);
    final SearchResponse decoratedSearchResponse = decoratorProcessor.decorateSearchResponse(searchResponse, searchType.decorators());
    final MessageList.Result.Builder resultBuilder = MessageList.Result.result(searchType.id()).messages(decoratedSearchResponse.messages()).effectiveTimerange(AbsoluteRange.create(from, to)).totalResults(decoratedSearchResponse.totalResults());
    return searchType.name().map(resultBuilder::name).orElse(resultBuilder).build();
}
Also used : ESGeneratedQueryContext(org.graylog.storage.elasticsearch6.views.ESGeneratedQueryContext) Query(org.graylog.plugins.views.search.Query) SearchResponse(org.graylog2.rest.resources.search.responses.SearchResponse) FieldSortBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.sort.FieldSortBuilder) SortOrder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.sort.SortOrder) ResultMessageSummary(org.graylog2.rest.models.messages.responses.ResultMessageSummary) Inject(javax.inject.Inject) SortBuilders(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.sort.SortBuilders) ResultMessage(org.graylog2.indexer.results.ResultMessage) SearchType(org.graylog.plugins.views.search.SearchType) Sort(org.graylog.plugins.views.search.searchtypes.Sort) Map(java.util.Map) AbsoluteRange(org.graylog2.plugin.indexer.searches.timeranges.AbsoluteRange) MessageList(org.graylog.plugins.views.search.searchtypes.MessageList) LegacyDecoratorProcessor(org.graylog.plugins.views.search.LegacyDecoratorProcessor) QueryBuilders(org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.QueryBuilders) HighlightBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.fetch.subphase.highlight.HighlightBuilder) SearchJob(org.graylog.plugins.views.search.SearchJob) DateTime(org.joda.time.DateTime) QueryStringQueryBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.index.query.QueryStringQueryBuilder) Set(java.util.Set) Collectors(java.util.stream.Collectors) List(java.util.List) QueryStringDecorators(org.graylog.plugins.views.search.elasticsearch.QueryStringDecorators) MetricAggregation(io.searchbox.core.search.aggregation.MetricAggregation) Optional(java.util.Optional) Named(com.google.inject.name.Named) MoreObjects.firstNonNull(com.google.common.base.MoreObjects.firstNonNull) VisibleForTesting(com.google.common.annotations.VisibleForTesting) Message(org.graylog2.plugin.Message) Collections(java.util.Collections) SearchSourceBuilder(org.graylog.shaded.elasticsearch6.org.elasticsearch.search.builder.SearchSourceBuilder) SearchResult(io.searchbox.core.SearchResult) Map(java.util.Map) DateTime(org.joda.time.DateTime) ResultMessageSummary(org.graylog2.rest.models.messages.responses.ResultMessageSummary) SearchResponse(org.graylog2.rest.resources.search.responses.SearchResponse) SearchResult(io.searchbox.core.SearchResult)

Example 15 with MetricAggregation

use of io.searchbox.core.search.aggregation.MetricAggregation in project graylog2-server by Graylog2.

the class ESPivot method processColumns.

private void processColumns(PivotResult.Row.Builder rowBuilder, SearchResult searchResult, ESGeneratedQueryContext queryContext, Pivot pivot, List<BucketSpec> remainingColumns, ArrayDeque<String> columnKeys, MetricAggregation aggregation) {
    if (remainingColumns.isEmpty()) {
        // with duplicate data entries
        if (!columnKeys.isEmpty()) {
            processSeries(rowBuilder, searchResult, queryContext, pivot, columnKeys, aggregation, false, "col-leaf");
        }
    } else {
        // for a non-leaf column group, we need to recurse further into the aggregation tree
        // and if rollup was requested we'll add intermediate series according to the column keys
        final BucketSpec currentBucket = remainingColumns.get(0);
        // this handler should never be missing, because we used it above to generate the query
        // if it is missing for some weird reason, it's ok to fail hard here
        final ESPivotBucketSpecHandler<? extends PivotSpec, ? extends Aggregation> handler = bucketHandlers.get(currentBucket.type());
        final Aggregation aggregationResult = handler.extractAggregationFromResult(pivot, currentBucket, aggregation, queryContext);
        final Stream<ESPivotBucketSpecHandler.Bucket> bucketStream = handler.handleResult(pivot, currentBucket, searchResult, aggregationResult, this, queryContext);
        // for each bucket, recurse and eventually collect all the column keys. once we reach a leaf, we'll end up in the other if branch above
        bucketStream.forEach(bucket -> {
            // push the bucket's key and use its aggregation as the new source for sub-aggregations
            columnKeys.addLast(bucket.key());
            processColumns(rowBuilder, searchResult, queryContext, pivot, tail(remainingColumns), columnKeys, bucket.aggregation());
            columnKeys.removeLast();
        });
        // don't add the empty column key rollup, because that's not the correct bucket here, it's being done in the row-leaf code
        if (pivot.rollup() && !columnKeys.isEmpty()) {
            // columnKeys is not empty, because this is a rollup per column in a row
            processSeries(rowBuilder, searchResult, queryContext, pivot, columnKeys, aggregation, true, "col-inner");
        }
    }
}
Also used : Aggregation(io.searchbox.core.search.aggregation.Aggregation) MetricAggregation(io.searchbox.core.search.aggregation.MetricAggregation) BucketSpec(org.graylog.plugins.views.search.searchtypes.pivot.BucketSpec)

Aggregations

MetricAggregation (io.searchbox.core.search.aggregation.MetricAggregation)18 Test (org.junit.Test)9 SearchResult (io.searchbox.core.SearchResult)7 TermsAggregation (io.searchbox.core.search.aggregation.TermsAggregation)7 SearchType (org.graylog.plugins.views.search.SearchType)7 PivotResult (org.graylog.plugins.views.search.searchtypes.pivot.PivotResult)5 ArrayList (java.util.ArrayList)4 Date (java.util.Date)4 Aggregation (io.searchbox.core.search.aggregation.Aggregation)3 List (java.util.List)3 Map (java.util.Map)3 AbstractDaoTest (org.finra.herd.dao.AbstractDaoTest)3 ResultTypeIndexSearchResponseDto (org.finra.herd.model.dto.ResultTypeIndexSearchResponseDto)3 TagTypeIndexSearchResponseDto (org.finra.herd.model.dto.TagTypeIndexSearchResponseDto)3 Query (org.graylog.plugins.views.search.Query)3 SearchJob (org.graylog.plugins.views.search.SearchJob)3 ESGeneratedQueryContext (org.graylog.storage.elasticsearch6.views.ESGeneratedQueryContext)3 Optional (java.util.Optional)2 Inject (javax.inject.Inject)2 BucketSpec (org.graylog.plugins.views.search.searchtypes.pivot.BucketSpec)2