Examples with InternalAggregations org.elasticsearch.search.aggregations.InternalAggregations used on opensource projects

Example 6 with InternalAggregations

use of org.elasticsearch.search.aggregations.InternalAggregations in project elasticsearch by elastic.

the class SearchPhaseControllerTests method testConsumerConcurrently.

public void testConsumerConcurrently() throws InterruptedException {
    int expectedNumResults = randomIntBetween(1, 100);
    int bufferSize = randomIntBetween(2, 200);
    SearchRequest request = new SearchRequest();
    request.source(new SearchSourceBuilder().aggregation(AggregationBuilders.avg("foo")));
    request.setBatchedReduceSize(bufferSize);
    InitialSearchPhase.SearchPhaseResults<QuerySearchResultProvider> consumer = searchPhaseController.newSearchPhaseResults(request, expectedNumResults);
    AtomicInteger max = new AtomicInteger();
    CountDownLatch latch = new CountDownLatch(expectedNumResults);
    for (int i = 0; i < expectedNumResults; i++) {
        int id = i;
        Thread t = new Thread(() -> {
            int number = randomIntBetween(1, 1000);
            max.updateAndGet(prev -> Math.max(prev, number));
            QuerySearchResult result = new QuerySearchResult(id, new SearchShardTarget("node", new Index("a", "b"), id));
            result.topDocs(new TopDocs(id, new ScoreDoc[0], 0.0F), new DocValueFormat[0]);
            InternalAggregations aggs = new InternalAggregations(Arrays.asList(new InternalMax("test", (double) number, DocValueFormat.RAW, Collections.emptyList(), Collections.emptyMap())));
            result.aggregations(aggs);
            consumer.consumeResult(id, result);
            latch.countDown();
        });
        t.start();
    }
    latch.await();
    SearchPhaseController.ReducedQueryPhase reduce = consumer.reduce();
    InternalMax internalMax = (InternalMax) reduce.aggregations.asList().get(0);
    assertEquals(max.get(), internalMax.getValue(), 0.0D);
}

Example 7 with InternalAggregations

use of org.elasticsearch.search.aggregations.InternalAggregations in project elasticsearch by elastic.

the class InternalDateHistogram method addEmptyBuckets.

private void addEmptyBuckets(List<Bucket> list, ReduceContext reduceContext) {
    Bucket lastBucket = null;
    ExtendedBounds bounds = emptyBucketInfo.bounds;
    ListIterator<Bucket> iter = list.listIterator();
    // first adding all the empty buckets *before* the actual data (based on th extended_bounds.min the user requested)
    InternalAggregations reducedEmptySubAggs = InternalAggregations.reduce(Collections.singletonList(emptyBucketInfo.subAggregations), reduceContext);
    if (bounds != null) {
        Bucket firstBucket = iter.hasNext() ? list.get(iter.nextIndex()) : null;
        if (firstBucket == null) {
            if (bounds.getMin() != null && bounds.getMax() != null) {
                long key = bounds.getMin();
                long max = bounds.getMax();
                while (key <= max) {
                    iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
                    key = nextKey(key).longValue();
                }
            }
        } else {
            if (bounds.getMin() != null) {
                long key = bounds.getMin();
                if (key < firstBucket.key) {
                    while (key < firstBucket.key) {
                        iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
                        key = nextKey(key).longValue();
                    }
                }
            }
        }
    }
    // e.g. if the data series is [1,2,3,7] there're 3 empty buckets that will be created for 4,5,6
    while (iter.hasNext()) {
        Bucket nextBucket = list.get(iter.nextIndex());
        if (lastBucket != null) {
            long key = nextKey(lastBucket.key).longValue();
            while (key < nextBucket.key) {
                iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
                key = nextKey(key).longValue();
            }
            assert key == nextBucket.key;
        }
        lastBucket = iter.next();
    }
    // finally, adding the empty buckets *after* the actual data (based on the extended_bounds.max requested by the user)
    if (bounds != null && lastBucket != null && bounds.getMax() != null && bounds.getMax() > lastBucket.key) {
        long key = emptyBucketInfo.rounding.nextRoundingValue(lastBucket.key);
        long max = bounds.getMax();
        while (key <= max) {
            iter.add(new InternalDateHistogram.Bucket(key, 0, keyed, format, reducedEmptySubAggs));
            key = emptyBucketInfo.rounding.nextRoundingValue(key);
        }
    }
}

Example 8 with InternalAggregations

use of org.elasticsearch.search.aggregations.InternalAggregations in project elasticsearch by elastic.

the class SearchPhaseController method reduceAggs.

private InternalAggregations reduceAggs(List<InternalAggregations> aggregationsList, List<SiblingPipelineAggregator> pipelineAggregators, ReduceContext reduceContext) {
    InternalAggregations aggregations = InternalAggregations.reduce(aggregationsList, reduceContext);
    if (pipelineAggregators != null) {
        List<InternalAggregation> newAggs = StreamSupport.stream(aggregations.spliterator(), false).map((p) -> (InternalAggregation) p).collect(Collectors.toList());
        for (SiblingPipelineAggregator pipelineAggregator : pipelineAggregators) {
            InternalAggregation newAgg = pipelineAggregator.doReduce(new InternalAggregations(newAggs), reduceContext);
            newAggs.add(newAgg);
        }
        return new InternalAggregations(newAggs);
    }
    return aggregations;
}

Example 9 with InternalAggregations

use of org.elasticsearch.search.aggregations.InternalAggregations in project elasticsearch by elastic.

the class SearchPhaseController method reducedQueryPhase.

/**
     * Reduces the given query results and consumes all aggregations and profile results.
     * @param queryResults a list of non-null query shard results
     * @param bufferdAggs a list of pre-collected / buffered aggregations. if this list is non-null all aggregations have been consumed
     *                    from all non-null query results.
     * @param numReducePhases the number of non-final reduce phases applied to the query results.
     * @see QuerySearchResult#consumeAggs()
     * @see QuerySearchResult#consumeProfileResult()
     */
private ReducedQueryPhase reducedQueryPhase(List<? extends AtomicArray.Entry<? extends QuerySearchResultProvider>> queryResults, List<InternalAggregations> bufferdAggs, int numReducePhases) {
    assert numReducePhases >= 0 : "num reduce phases must be >= 0 but was: " + numReducePhases;
    // increment for this phase
    numReducePhases++;
    long totalHits = 0;
    long fetchHits = 0;
    float maxScore = Float.NEGATIVE_INFINITY;
    boolean timedOut = false;
    Boolean terminatedEarly = null;
    if (queryResults.isEmpty()) {
        // early terminate we have nothing to reduce
        return new ReducedQueryPhase(totalHits, fetchHits, maxScore, timedOut, terminatedEarly, null, null, null, null, numReducePhases);
    }
    final QuerySearchResult firstResult = queryResults.get(0).value.queryResult();
    final boolean hasSuggest = firstResult.suggest() != null;
    final boolean hasProfileResults = firstResult.hasProfileResults();
    final boolean consumeAggs;
    final List<InternalAggregations> aggregationsList;
    if (bufferdAggs != null) {
        consumeAggs = false;
        // we already have results from intermediate reduces and just need to perform the final reduce
        assert firstResult.hasAggs() : "firstResult has no aggs but we got non null buffered aggs?";
        aggregationsList = bufferdAggs;
    } else if (firstResult.hasAggs()) {
        // the number of shards was less than the buffer size so we reduce agg results directly
        aggregationsList = new ArrayList<>(queryResults.size());
        consumeAggs = true;
    } else {
        // no aggregations
        aggregationsList = Collections.emptyList();
        consumeAggs = false;
    }
    // count the total (we use the query result provider here, since we might not get any hits (we scrolled past them))
    final Map<String, List<Suggestion>> groupedSuggestions = hasSuggest ? new HashMap<>() : Collections.emptyMap();
    final Map<String, ProfileShardResult> profileResults = hasProfileResults ? new HashMap<>(queryResults.size()) : Collections.emptyMap();
    for (AtomicArray.Entry<? extends QuerySearchResultProvider> entry : queryResults) {
        QuerySearchResult result = entry.value.queryResult();
        if (result.searchTimedOut()) {
            timedOut = true;
        }
        if (result.terminatedEarly() != null) {
            if (terminatedEarly == null) {
                terminatedEarly = result.terminatedEarly();
            } else if (result.terminatedEarly()) {
                terminatedEarly = true;
            }
        }
        totalHits += result.topDocs().totalHits;
        fetchHits += result.topDocs().scoreDocs.length;
        if (!Float.isNaN(result.topDocs().getMaxScore())) {
            maxScore = Math.max(maxScore, result.topDocs().getMaxScore());
        }
        if (hasSuggest) {
            assert result.suggest() != null;
            for (Suggestion<? extends Suggestion.Entry<? extends Suggestion.Entry.Option>> suggestion : result.suggest()) {
                List<Suggestion> suggestionList = groupedSuggestions.computeIfAbsent(suggestion.getName(), s -> new ArrayList<>());
                suggestionList.add(suggestion);
            }
        }
        if (consumeAggs) {
            aggregationsList.add((InternalAggregations) result.consumeAggs());
        }
        if (hasProfileResults) {
            String key = result.shardTarget().toString();
            profileResults.put(key, result.consumeProfileResult());
        }
    }
    final Suggest suggest = groupedSuggestions.isEmpty() ? null : new Suggest(Suggest.reduce(groupedSuggestions));
    ReduceContext reduceContext = new ReduceContext(bigArrays, scriptService, true);
    final InternalAggregations aggregations = aggregationsList.isEmpty() ? null : reduceAggs(aggregationsList, firstResult.pipelineAggregators(), reduceContext);
    final SearchProfileShardResults shardResults = profileResults.isEmpty() ? null : new SearchProfileShardResults(profileResults);
    return new ReducedQueryPhase(totalHits, fetchHits, maxScore, timedOut, terminatedEarly, firstResult, suggest, aggregations, shardResults, numReducePhases);
}

Example 10 with InternalAggregations

use of org.elasticsearch.search.aggregations.InternalAggregations in project elasticsearch by elastic.

the class SerialDiffPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    MultiBucketsAggregation histo = (MultiBucketsAggregation) aggregation;
    List<? extends Bucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>();
    EvictingQueue<Double> lagWindow = new EvictingQueue<>(lag);
    int counter = 0;
    for (Bucket bucket : buckets) {
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
        Bucket newBucket = bucket;
        counter += 1;
        // Still under the initial lag period, add nothing and move on
        Double lagValue;
        if (counter <= lag) {
            lagValue = Double.NaN;
        } else {
            // Peek here, because we rely on add'ing to always move the window
            lagValue = lagWindow.peek();
        }
        // Normalize null's to NaN
        if (thisBucketValue == null) {
            thisBucketValue = Double.NaN;
        }
        // Both have values, calculate diff and replace the "empty" bucket
        if (!Double.isNaN(thisBucketValue) && !Double.isNaN(lagValue)) {
            double diff = thisBucketValue - lagValue;
            List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
                return (InternalAggregation) p;
            }).collect(Collectors.toList());
            aggs.add(new InternalSimpleValue(name(), diff, formatter, new ArrayList<PipelineAggregator>(), metaData()));
            newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), new InternalAggregations(aggs));
        }
        newBuckets.add(newBucket);
        lagWindow.add(thisBucketValue);
    }
    return factory.createAggregation(newBuckets);
}