Search in sources :

Example 1 with InternalMultiBucketAggregation

use of org.opensearch.search.aggregations.InternalMultiBucketAggregation in project OpenSearch by opensearch-project.

the class BucketSelectorPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket> originalAgg = (InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket>) aggregation;
    List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = originalAgg.getBuckets();
    BucketAggregationSelectorScript.Factory factory = reduceContext.scriptService().compile(script, BucketAggregationSelectorScript.CONTEXT);
    List<InternalMultiBucketAggregation.InternalBucket> newBuckets = new ArrayList<>();
    for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
        Map<String, Object> vars = new HashMap<>();
        if (script.getParams() != null) {
            vars.putAll(script.getParams());
        }
        for (Map.Entry<String, String> entry : bucketsPathsMap.entrySet()) {
            String varName = entry.getKey();
            String bucketsPath = entry.getValue();
            Double value = resolveBucketValue(originalAgg, bucket, bucketsPath, gapPolicy);
            vars.put(varName, value);
        }
        // TODO: can we use one instance of the script for all buckets? it should be stateless?
        BucketAggregationSelectorScript executableScript = factory.newInstance(vars);
        if (executableScript.execute()) {
            newBuckets.add(bucket);
        }
    }
    return originalAgg.create(newBuckets);
}
Also used : BucketAggregationSelectorScript(org.opensearch.script.BucketAggregationSelectorScript) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation) HashMap(java.util.HashMap) Map(java.util.Map)

Example 2 with InternalMultiBucketAggregation

use of org.opensearch.search.aggregations.InternalMultiBucketAggregation in project OpenSearch by opensearch-project.

the class CumulativeSumPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket> histo = (InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket>) aggregation;
    List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>(buckets.size());
    double sum = 0;
    for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], GapPolicy.INSERT_ZEROS);
        // Only increment the sum if it's a finite value, otherwise "increment by zero" is correct
        if (thisBucketValue != null && thisBucketValue.isInfinite() == false && thisBucketValue.isNaN() == false) {
            sum += thisBucketValue;
        }
        List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> (InternalAggregation) p).collect(Collectors.toList());
        aggs.add(new InternalSimpleValue(name(), sum, formatter, metadata()));
        Bucket newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), InternalAggregations.from(aggs));
        newBuckets.add(newBucket);
    }
    return factory.createAggregation(newBuckets);
}
Also used : StreamInput(org.opensearch.common.io.stream.StreamInput) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) DocValueFormat(org.opensearch.search.DocValueFormat) ReduceContext(org.opensearch.search.aggregations.InternalAggregation.ReduceContext) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation) StreamOutput(org.opensearch.common.io.stream.StreamOutput) IOException(java.io.IOException) GapPolicy(org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy) Collectors(java.util.stream.Collectors) ArrayList(java.util.ArrayList) List(java.util.List) BucketHelpers.resolveBucketValue(org.opensearch.search.aggregations.pipeline.BucketHelpers.resolveBucketValue) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) Map(java.util.Map) StreamSupport(java.util.stream.StreamSupport) Bucket(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) ArrayList(java.util.ArrayList) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) Bucket(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation)

Example 3 with InternalMultiBucketAggregation

use of org.opensearch.search.aggregations.InternalMultiBucketAggregation in project OpenSearch by opensearch-project.

the class MovAvgPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket> histo = (InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket>) aggregation;
    List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>();
    EvictingQueue<Double> values = new EvictingQueue<>(this.window);
    Number lastValidKey = 0;
    int lastValidPosition = 0;
    int counter = 0;
    // Do we need to fit the model parameters to the data?
    if (minimize) {
        assert (model.canBeMinimized());
        model = minimize(buckets, histo, model);
    }
    for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
        // Default is to reuse existing bucket. Simplifies the rest of the logic,
        // since we only change newBucket if we can add to it
        Bucket newBucket = bucket;
        if ((thisBucketValue == null || thisBucketValue.equals(Double.NaN)) == false) {
            // Some models (e.g. HoltWinters) have certain preconditions that must be met
            if (model.hasValue(values.size())) {
                double movavg = model.next(values);
                List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> (InternalAggregation) p).collect(Collectors.toList());
                aggs.add(new InternalSimpleValue(name(), movavg, formatter, metadata()));
                newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), InternalAggregations.from(aggs));
            }
            if (predict > 0) {
                lastValidKey = factory.getKey(bucket);
                lastValidPosition = counter;
            }
            values.offer(thisBucketValue);
        }
        counter += 1;
        newBuckets.add(newBucket);
    }
    if (buckets.size() > 0 && predict > 0) {
        double[] predictions = model.predict(values, predict);
        for (int i = 0; i < predictions.length; i++) {
            List<InternalAggregation> aggs;
            Number newKey = factory.nextKey(lastValidKey);
            if (lastValidPosition + i + 1 < newBuckets.size()) {
                Bucket bucket = newBuckets.get(lastValidPosition + i + 1);
                // Get the existing aggs in the bucket so we don't clobber data
                aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> (InternalAggregation) p).collect(Collectors.toList());
                aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, metadata()));
                Bucket newBucket = factory.createBucket(newKey, bucket.getDocCount(), InternalAggregations.from(aggs));
                // Overwrite the existing bucket with the new version
                newBuckets.set(lastValidPosition + i + 1, newBucket);
            } else {
                // Not seen before, create fresh
                aggs = new ArrayList<>();
                aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, metadata()));
                Bucket newBucket = factory.createBucket(newKey, 0, InternalAggregations.from(aggs));
                // Since this is a new bucket, simply append it
                newBuckets.add(newBucket);
            }
            lastValidKey = newKey;
        }
    }
    return factory.createAggregation(newBuckets);
}
Also used : StreamInput(org.opensearch.common.io.stream.StreamInput) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) ListIterator(java.util.ListIterator) DocValueFormat(org.opensearch.search.DocValueFormat) ReduceContext(org.opensearch.search.aggregations.InternalAggregation.ReduceContext) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation) EvictingQueue(org.opensearch.common.collect.EvictingQueue) StreamOutput(org.opensearch.common.io.stream.StreamOutput) IOException(java.io.IOException) MultiBucketsAggregation(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation) GapPolicy(org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy) Collectors(java.util.stream.Collectors) ArrayList(java.util.ArrayList) List(java.util.List) BucketHelpers.resolveBucketValue(org.opensearch.search.aggregations.pipeline.BucketHelpers.resolveBucketValue) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) Map(java.util.Map) StreamSupport(java.util.stream.StreamSupport) Bucket(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) ArrayList(java.util.ArrayList) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) Bucket(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket) EvictingQueue(org.opensearch.common.collect.EvictingQueue) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation)

Example 4 with InternalMultiBucketAggregation

use of org.opensearch.search.aggregations.InternalMultiBucketAggregation in project OpenSearch by opensearch-project.

the class TermsAggregatorTests method testWithNestedAggregations.

public void testWithNestedAggregations() throws IOException {
    try (Directory directory = newDirectory()) {
        try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) {
            for (int i = 0; i < 10; i++) {
                int[] nestedValues = new int[i];
                for (int j = 0; j < i; j++) {
                    nestedValues[j] = j;
                }
                indexWriter.addDocuments(generateDocsWithNested(Integer.toString(i), i, nestedValues));
            }
            indexWriter.commit();
            for (Aggregator.SubAggCollectionMode mode : Aggregator.SubAggCollectionMode.values()) {
                for (boolean withScore : new boolean[] { true, false }) {
                    NestedAggregationBuilder nested = new NestedAggregationBuilder("nested", "nested_object").subAggregation(new TermsAggregationBuilder("terms").userValueTypeHint(ValueType.LONG).field("nested_value").collectMode(mode).order(BucketOrder.key(true)).subAggregation(new TopHitsAggregationBuilder("top_hits").sort(withScore ? new ScoreSortBuilder() : new FieldSortBuilder("_doc")).storedField("_none_")));
                    MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType("nested_value", NumberFieldMapper.NumberType.LONG);
                    try (IndexReader indexReader = wrapInMockESDirectoryReader(DirectoryReader.open(directory))) {
                        {
                            InternalNested result = searchAndReduce(newSearcher(indexReader, false, true), // match root document only
                            new DocValuesFieldExistsQuery(PRIMARY_TERM_NAME), nested, fieldType);
                            InternalMultiBucketAggregation<?, ?> terms = result.getAggregations().get("terms");
                            assertNestedTopHitsScore(terms, withScore);
                        }
                        {
                            FilterAggregationBuilder filter = new FilterAggregationBuilder("filter", new MatchAllQueryBuilder()).subAggregation(nested);
                            InternalFilter result = searchAndReduce(newSearcher(indexReader, false, true), // match root document only
                            new DocValuesFieldExistsQuery(PRIMARY_TERM_NAME), filter, fieldType);
                            InternalNested nestedResult = result.getAggregations().get("nested");
                            InternalMultiBucketAggregation<?, ?> terms = nestedResult.getAggregations().get("terms");
                            assertNestedTopHitsScore(terms, withScore);
                        }
                    }
                }
            }
        }
    }
}
Also used : FilterAggregationBuilder(org.opensearch.search.aggregations.bucket.filter.FilterAggregationBuilder) TopHitsAggregationBuilder(org.opensearch.search.aggregations.metrics.TopHitsAggregationBuilder) InternalNested(org.opensearch.search.aggregations.bucket.nested.InternalNested) Aggregator(org.opensearch.search.aggregations.Aggregator) FieldSortBuilder(org.opensearch.search.sort.FieldSortBuilder) DocValuesFieldExistsQuery(org.apache.lucene.search.DocValuesFieldExistsQuery) InetAddressPoint(org.apache.lucene.document.InetAddressPoint) GeoPoint(org.opensearch.common.geo.GeoPoint) ScoreSortBuilder(org.opensearch.search.sort.ScoreSortBuilder) InternalFilter(org.opensearch.search.aggregations.bucket.filter.InternalFilter) MappedFieldType(org.opensearch.index.mapper.MappedFieldType) IndexReader(org.apache.lucene.index.IndexReader) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation) RandomIndexWriter(org.apache.lucene.index.RandomIndexWriter) Directory(org.apache.lucene.store.Directory) NestedAggregationBuilder(org.opensearch.search.aggregations.bucket.nested.NestedAggregationBuilder) MatchAllQueryBuilder(org.opensearch.index.query.MatchAllQueryBuilder)

Example 5 with InternalMultiBucketAggregation

use of org.opensearch.search.aggregations.InternalMultiBucketAggregation in project OpenSearch by opensearch-project.

the class MovFnPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, InternalAggregation.ReduceContext reduceContext) {
    InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket> histo = (InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket>) aggregation;
    List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<MultiBucketsAggregation.Bucket> newBuckets = new ArrayList<>();
    // Initialize the script
    MovingFunctionScript.Factory scriptFactory = reduceContext.scriptService().compile(script, MovingFunctionScript.CONTEXT);
    Map<String, Object> vars = new HashMap<>();
    if (script.getParams() != null) {
        vars.putAll(script.getParams());
    }
    MovingFunctionScript executableScript = scriptFactory.newInstance();
    List<Double> values = buckets.stream().map(b -> resolveBucketValue(histo, b, bucketsPaths()[0], gapPolicy)).filter(v -> v != null && v.isNaN() == false).collect(Collectors.toList());
    int index = 0;
    for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
        // Default is to reuse existing bucket. Simplifies the rest of the logic,
        // since we only change newBucket if we can add to it
        MultiBucketsAggregation.Bucket newBucket = bucket;
        if (thisBucketValue != null && thisBucketValue.isNaN() == false) {
            // The custom context mandates that the script returns a double (not Double) so we
            // don't need null checks, etc.
            int fromIndex = clamp(index - window + shift, values);
            int toIndex = clamp(index + shift, values);
            double movavg = executableScript.execute(vars, values.subList(fromIndex, toIndex).stream().mapToDouble(Double::doubleValue).toArray());
            List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map(InternalAggregation.class::cast).collect(Collectors.toList());
            aggs.add(new InternalSimpleValue(name(), movavg, formatter, metadata()));
            newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), InternalAggregations.from(aggs));
            index++;
        }
        newBuckets.add(newBucket);
    }
    return factory.createAggregation(newBuckets);
}
Also used : StreamInput(org.opensearch.common.io.stream.StreamInput) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) DocValueFormat(org.opensearch.search.DocValueFormat) Script(org.opensearch.script.Script) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation) StreamOutput(org.opensearch.common.io.stream.StreamOutput) IOException(java.io.IOException) HashMap(java.util.HashMap) MultiBucketsAggregation(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation) Collectors(java.util.stream.Collectors) ArrayList(java.util.ArrayList) List(java.util.List) BucketHelpers.resolveBucketValue(org.opensearch.search.aggregations.pipeline.BucketHelpers.resolveBucketValue) LegacyESVersion(org.opensearch.LegacyESVersion) InternalAggregations(org.opensearch.search.aggregations.InternalAggregations) Map(java.util.Map) StreamSupport(java.util.stream.StreamSupport) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) HashMap(java.util.HashMap) ArrayList(java.util.ArrayList) HistogramFactory(org.opensearch.search.aggregations.bucket.histogram.HistogramFactory) InternalMultiBucketAggregation(org.opensearch.search.aggregations.InternalMultiBucketAggregation) InternalAggregation(org.opensearch.search.aggregations.InternalAggregation) MultiBucketsAggregation(org.opensearch.search.aggregations.bucket.MultiBucketsAggregation)

Aggregations

InternalMultiBucketAggregation (org.opensearch.search.aggregations.InternalMultiBucketAggregation)8 ArrayList (java.util.ArrayList)6 Map (java.util.Map)6 InternalAggregation (org.opensearch.search.aggregations.InternalAggregation)6 IOException (java.io.IOException)5 List (java.util.List)5 Collectors (java.util.stream.Collectors)5 StreamSupport (java.util.stream.StreamSupport)5 StreamInput (org.opensearch.common.io.stream.StreamInput)5 StreamOutput (org.opensearch.common.io.stream.StreamOutput)5 DocValueFormat (org.opensearch.search.DocValueFormat)5 InternalAggregations (org.opensearch.search.aggregations.InternalAggregations)5 HistogramFactory (org.opensearch.search.aggregations.bucket.histogram.HistogramFactory)5 BucketHelpers.resolveBucketValue (org.opensearch.search.aggregations.pipeline.BucketHelpers.resolveBucketValue)5 ReduceContext (org.opensearch.search.aggregations.InternalAggregation.ReduceContext)4 Bucket (org.opensearch.search.aggregations.bucket.MultiBucketsAggregation.Bucket)4 GapPolicy (org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy)4 HashMap (java.util.HashMap)2 EvictingQueue (org.opensearch.common.collect.EvictingQueue)2 MultiBucketsAggregation (org.opensearch.search.aggregations.bucket.MultiBucketsAggregation)2