Search in sources :

Example 1 with ReduceContext

use of org.opensearch.search.aggregations.InternalAggregation.ReduceContext in project OpenSearch by opensearch-project.

the class CumulativeSumPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket> histo = (InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket>) aggregation;
    List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>(buckets.size());
    double sum = 0;
    for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], GapPolicy.INSERT_ZEROS);
        // Only increment the sum if it's a finite value, otherwise "increment by zero" is correct
        if (thisBucketValue != null && thisBucketValue.isInfinite() == false && thisBucketValue.isNaN() == false) {
            sum += thisBucketValue;
        }
        List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> (InternalAggregation) p).collect(Collectors.toList());
        aggs.add(new InternalSimpleValue(name(), sum, formatter, metadata()));
        Bucket newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), InternalAggregations.from(aggs));
        newBuckets.add(newBucket);
    }
    return factory.createAggregation(newBuckets);
}
Also used : StreamInput(org.opensearch.common.io.stream.StreamInput) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) DocValueFormat(org.opensearch.search.DocValueFormat) ReduceContext(org.opensearch.search.aggregations.InternalAggregation.ReduceContext) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation) StreamOutput(org.opensearch.common.io.stream.StreamOutput) IOException(java.io.IOException) GapPolicy(org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy) Collectors(java.util.stream.Collectors) ArrayList(java.util.ArrayList) List(java.util.List) BucketHelpers.resolveBucketValue(org.opensearch.search.aggregations.pipeline.BucketHelpers.resolveBucketValue) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) Map(java.util.Map) StreamSupport(java.util.stream.StreamSupport) Bucket(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) ArrayList(java.util.ArrayList) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) Bucket(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation)

Example 2 with ReduceContext

use of org.opensearch.search.aggregations.InternalAggregation.ReduceContext in project OpenSearch by opensearch-project.

the class MovAvgPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket> histo = (InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket>) aggregation;
    List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>();
    EvictingQueue<Double> values = new EvictingQueue<>(this.window);
    Number lastValidKey = 0;
    int lastValidPosition = 0;
    int counter = 0;
    // Do we need to fit the model parameters to the data?
    if (minimize) {
        assert (model.canBeMinimized());
        model = minimize(buckets, histo, model);
    }
    for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
        // Default is to reuse existing bucket. Simplifies the rest of the logic,
        // since we only change newBucket if we can add to it
        Bucket newBucket = bucket;
        if ((thisBucketValue == null || thisBucketValue.equals(Double.NaN)) == false) {
            // Some models (e.g. HoltWinters) have certain preconditions that must be met
            if (model.hasValue(values.size())) {
                double movavg = model.next(values);
                List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> (InternalAggregation) p).collect(Collectors.toList());
                aggs.add(new InternalSimpleValue(name(), movavg, formatter, metadata()));
                newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), InternalAggregations.from(aggs));
            }
            if (predict > 0) {
                lastValidKey = factory.getKey(bucket);
                lastValidPosition = counter;
            }
            values.offer(thisBucketValue);
        }
        counter += 1;
        newBuckets.add(newBucket);
    }
    if (buckets.size() > 0 && predict > 0) {
        double[] predictions = model.predict(values, predict);
        for (int i = 0; i < predictions.length; i++) {
            List<InternalAggregation> aggs;
            Number newKey = factory.nextKey(lastValidKey);
            if (lastValidPosition + i + 1 < newBuckets.size()) {
                Bucket bucket = newBuckets.get(lastValidPosition + i + 1);
                // Get the existing aggs in the bucket so we don't clobber data
                aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> (InternalAggregation) p).collect(Collectors.toList());
                aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, metadata()));
                Bucket newBucket = factory.createBucket(newKey, bucket.getDocCount(), InternalAggregations.from(aggs));
                // Overwrite the existing bucket with the new version
                newBuckets.set(lastValidPosition + i + 1, newBucket);
            } else {
                // Not seen before, create fresh
                aggs = new ArrayList<>();
                aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, metadata()));
                Bucket newBucket = factory.createBucket(newKey, 0, InternalAggregations.from(aggs));
                // Since this is a new bucket, simply append it
                newBuckets.add(newBucket);
            }
            lastValidKey = newKey;
        }
    }
    return factory.createAggregation(newBuckets);
}
Also used : StreamInput(org.opensearch.common.io.stream.StreamInput) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) ListIterator(java.util.ListIterator) DocValueFormat(org.opensearch.search.DocValueFormat) ReduceContext(org.opensearch.search.aggregations.InternalAggregation.ReduceContext) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation) EvictingQueue(org.opensearch.common.collect.EvictingQueue) StreamOutput(org.opensearch.common.io.stream.StreamOutput) IOException(java.io.IOException) MultiBucketsAggregation(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation) GapPolicy(org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy) Collectors(java.util.stream.Collectors) ArrayList(java.util.ArrayList) List(java.util.List) BucketHelpers.resolveBucketValue(org.opensearch.search.aggregations.pipeline.BucketHelpers.resolveBucketValue) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) Map(java.util.Map) StreamSupport(java.util.stream.StreamSupport) Bucket(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) ArrayList(java.util.ArrayList) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) Bucket(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket) EvictingQueue(org.opensearch.common.collect.EvictingQueue) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation)

Example 3 with ReduceContext

use of org.opensearch.search.aggregations.InternalAggregation.ReduceContext in project OpenSearch by opensearch-project.

the class SearchPhaseControllerTests method setup.

@Before
public void setup() {
    reductions = new CopyOnWriteArrayList<>();
    searchPhaseController = new SearchPhaseController(writableRegistry(), s -> new InternalAggregation.ReduceContextBuilder() {

        @Override
        public ReduceContext forPartialReduction() {
            reductions.add(false);
            return InternalAggregation.ReduceContext.forPartialReduction(BigArrays.NON_RECYCLING_INSTANCE, null, () -> PipelineTree.EMPTY);
        }

        public ReduceContext forFinalReduction() {
            reductions.add(true);
            return InternalAggregation.ReduceContext.forFinalReduction(BigArrays.NON_RECYCLING_INSTANCE, null, b -> {
            }, PipelineTree.EMPTY);
        }
    });
    threadPool = new TestThreadPool(SearchPhaseControllerTests.class.getName());
    fixedExecutor = OpenSearchExecutors.newFixed("test", 1, 10, OpenSearchExecutors.daemonThreadFactory("test"), threadPool.getThreadContext());
}
Also used : InternalAggregationTestCase(org.opensearch.test.InternalAggregationTestCase) ScoreDoc(org.apache.lucene.search.ScoreDoc) ShardSearchContextId(org.opensearch.search.internal.ShardSearchContextId) SearchContext(org.opensearch.search.internal.SearchContext) TestThreadPool(org.opensearch.threadpool.TestThreadPool) FieldDoc(org.apache.lucene.search.FieldDoc) CircuitBreaker(org.opensearch.common.breaker.CircuitBreaker) SortBy(org.opensearch.search.suggest.SortBy) Strings(org.opensearch.common.Strings) Collections.singletonList(java.util.Collections.singletonList) InternalMax(org.opensearch.search.aggregations.metrics.InternalMax) CollapseTopFieldDocs(org.apache.lucene.search.grouping.CollapseTopFieldDocs) AtomicInteger(java.util.concurrent.atomic.AtomicInteger) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) After(org.junit.After) Map(java.util.Map) Lucene(org.opensearch.common.lucene.Lucene) SortField(org.apache.lucene.search.SortField) RandomizedContext(com.carrotsearch.randomizedtesting.RandomizedContext) BytesRef(org.apache.lucene.util.BytesRef) SearchHit(org.opensearch.search.SearchHit) Collections.emptyList(java.util.Collections.emptyList) Matchers.lessThanOrEqualTo(org.hamcrest.Matchers.lessThanOrEqualTo) OpenSearchTestCase(org.opensearch.test.OpenSearchTestCase) Settings(org.opensearch.common.settings.Settings) CompletionSuggestion(org.opensearch.search.suggest.completion.CompletionSuggestion) PhraseSuggestion(org.opensearch.search.suggest.phrase.PhraseSuggestion) Collectors(java.util.stream.Collectors) OriginalIndices(org.opensearch.action.OriginalIndices) OpenSearchThreadPoolExecutor(org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor) Matchers.instanceOf(org.hamcrest.Matchers.instanceOf) CountDownLatch(java.util.concurrent.CountDownLatch) List(java.util.List) Stream(java.util.stream.Stream) SearchPhaseResult(org.opensearch.search.SearchPhaseResult) SearchSourceBuilder(org.opensearch.search.builder.SearchSourceBuilder) Matchers.equalTo(org.hamcrest.Matchers.equalTo) Optional(java.util.Optional) TopFieldDocs(org.apache.lucene.search.TopFieldDocs) BigArrays(org.opensearch.common.util.BigArrays) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) Matchers.containsString(org.hamcrest.Matchers.containsString) CopyOnWriteArrayList(java.util.concurrent.CopyOnWriteArrayList) TermSuggestion(org.opensearch.search.suggest.term.TermSuggestion) DocValueFormat(org.opensearch.search.DocValueFormat) ReduceContext(org.opensearch.search.aggregations.InternalAggregation.ReduceContext) ThreadPool(org.opensearch.threadpool.ThreadPool) AtomicBoolean(java.util.concurrent.atomic.AtomicBoolean) HashMap(java.util.HashMap) SearchHits(org.opensearch.search.SearchHits) QuerySearchResult(org.opensearch.search.query.QuerySearchResult) OpenSearchExecutors(org.opensearch.common.util.concurrent.OpenSearchExecutors) AtomicReference(java.util.concurrent.atomic.AtomicReference) NamedWriteableRegistry(org.opensearch.common.io.stream.NamedWriteableRegistry) ArrayList(java.util.ArrayList) PipelineTree(org.opensearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree) Matchers.lessThan(org.hamcrest.Matchers.lessThan) UUIDs(org.opensearch.common.UUIDs) Before(org.junit.Before) CircuitBreakingException(org.opensearch.common.breaker.CircuitBreakingException) TopDocs(org.apache.lucene.search.TopDocs) NoopCircuitBreaker(org.opensearch.common.breaker.NoopCircuitBreaker) Collections.emptyMap(java.util.Collections.emptyMap) Matchers.greaterThanOrEqualTo(org.hamcrest.Matchers.greaterThanOrEqualTo) Relation(org.apache.lucene.search.TotalHits.Relation) InternalSearchResponse(org.opensearch.search.internal.InternalSearchResponse) TotalHits(org.apache.lucene.search.TotalHits) ShardId(org.opensearch.index.shard.ShardId) AggregationBuilders(org.opensearch.search.aggregations.AggregationBuilders) AtomicArray(org.opensearch.common.util.concurrent.AtomicArray) SearchShardTarget(org.opensearch.search.SearchShardTarget) TopDocsAndMaxScore(org.opensearch.common.lucene.search.TopDocsAndMaxScore) Suggest(org.opensearch.search.suggest.Suggest) FetchSearchResult(org.opensearch.search.fetch.FetchSearchResult) Collections(java.util.Collections) Text(org.opensearch.common.text.Text) SearchModule(org.opensearch.search.SearchModule) TestThreadPool(org.opensearch.threadpool.TestThreadPool) Before(org.junit.Before)

Example 4 with ReduceContext

use of org.opensearch.search.aggregations.InternalAggregation.ReduceContext in project OpenSearch by opensearch-project.

the class InternalFilterTests method testReducePipelinesReducesBucketPipelines.

public void testReducePipelinesReducesBucketPipelines() {
    /*
         * Tests that a pipeline buckets by creating a mock pipeline that
         * replaces "inner" with "dummy".
         */
    InternalFilter dummy = createTestInstance();
    InternalFilter inner = createTestInstance();
    InternalAggregations sub = InternalAggregations.from(Collections.singletonList(inner));
    InternalFilter test = createTestInstance("test", randomNonNegativeLong(), sub, emptyMap());
    PipelineAggregator mockPipeline = new PipelineAggregator(null, null, null) {

        @Override
        public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
            return dummy;
        }
    };
    PipelineTree tree = new PipelineTree(org.opensearch.common.collect.Map.of(inner.getName(), new PipelineTree(emptyMap(), singletonList(mockPipeline))), emptyList());
    InternalFilter reduced = (InternalFilter) test.reducePipelines(test, emptyReduceContextBuilder().forFinalReduction(), tree);
    assertThat(reduced.getAggregations().get(dummy.getName()), sameInstance(dummy));
}
Also used : InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) PipelineAggregator(org.opensearch.search.aggregations.pipeline.PipelineAggregator) ReduceContext(org.opensearch.search.aggregations.InternalAggregation.ReduceContext) PipelineTree(org.opensearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree)

Example 5 with ReduceContext

use of org.opensearch.search.aggregations.InternalAggregation.ReduceContext in project OpenSearch by opensearch-project.

the class SerialDiffPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket> histo = (InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket>) aggregation;
    List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>();
    EvictingQueue<Double> lagWindow = new EvictingQueue<>(lag);
    int counter = 0;
    for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
        Bucket newBucket = bucket;
        counter += 1;
        // Still under the initial lag period, add nothing and move on
        Double lagValue;
        if (counter <= lag) {
            lagValue = Double.NaN;
        } else {
            // Peek here, because we rely on add'ing to always move the window
            lagValue = lagWindow.peek();
        }
        // Normalize null's to NaN
        if (thisBucketValue == null) {
            thisBucketValue = Double.NaN;
        }
        // Both have values, calculate diff and replace the "empty" bucket
        if (!Double.isNaN(thisBucketValue) && !Double.isNaN(lagValue)) {
            double diff = thisBucketValue - lagValue;
            List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> (InternalAggregation) p).collect(Collectors.toList());
            aggs.add(new InternalSimpleValue(name(), diff, formatter, metadata()));
            newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), InternalAggregations.from(aggs));
        }
        newBuckets.add(newBucket);
        lagWindow.add(thisBucketValue);
    }
    return factory.createAggregation(newBuckets);
}
Also used : StreamInput(org.opensearch.common.io.stream.StreamInput) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) DocValueFormat(org.opensearch.search.DocValueFormat) ReduceContext(org.opensearch.search.aggregations.InternalAggregation.ReduceContext) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation) EvictingQueue(org.opensearch.common.collect.EvictingQueue) StreamOutput(org.opensearch.common.io.stream.StreamOutput) IOException(java.io.IOException) GapPolicy(org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy) Collectors(java.util.stream.Collectors) Nullable(org.opensearch.common.Nullable) ArrayList(java.util.ArrayList) List(java.util.List) BucketHelpers.resolveBucketValue(org.opensearch.search.aggregations.pipeline.BucketHelpers.resolveBucketValue) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) Map(java.util.Map) StreamSupport(java.util.stream.StreamSupport) Bucket(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) ArrayList(java.util.ArrayList) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) Bucket(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket) EvictingQueue(org.opensearch.common.collect.EvictingQueue) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation)

Aggregations

InternalAggregation (org.opensearch.search.aggregations.InternalAggregation)7 ReduceContext (org.opensearch.search.aggregations.InternalAggregation.ReduceContext)7 InternalAggregations (org.opensearch.search.aggregations.InternalAggregations)7 ArrayList (java.util.ArrayList)5 List (java.util.List)5 Map (java.util.Map)5 Collectors (java.util.stream.Collectors)5 DocValueFormat (org.opensearch.search.DocValueFormat)5 IOException (java.io.IOException)4 StreamSupport (java.util.stream.StreamSupport)4 StreamInput (org.opensearch.common.io.stream.StreamInput)4 StreamOutput (org.opensearch.common.io.stream.StreamOutput)4 InternalMultiBucketAggregation (org.opensearch.search.aggregations.InternalMultiBucketAggregation)4 Bucket (org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket)4 HistogramFactory (org.opensearch.search.aggregations.bucket.histogram.HistogramFactory)4 GapPolicy (org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy)4 BucketHelpers.resolveBucketValue (org.opensearch.search.aggregations.pipeline.BucketHelpers.resolveBucketValue)4 PipelineTree (org.opensearch.search.aggregations.pipeline.PipelineAggregator.PipelineTree)3 RandomizedContext (com.carrotsearch.randomizedtesting.RandomizedContext)1 Collections (java.util.Collections)1