Examples with PipelineAggregator org.elasticsearch.search.aggregations.pipeline.PipelineAggregator used on opensource projects

Example 1 with PipelineAggregator

use of org.elasticsearch.search.aggregations.pipeline.PipelineAggregator in project elasticsearch by elastic.

the class CumulativeSumPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    MultiBucketsAggregation histo = (MultiBucketsAggregation) aggregation;
    List<? extends Bucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>();
    double sum = 0;
    for (Bucket bucket : buckets) {
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], GapPolicy.INSERT_ZEROS);
        sum += thisBucketValue;
        List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
            return (InternalAggregation) p;
        }).collect(Collectors.toList());
        aggs.add(new InternalSimpleValue(name(), sum, formatter, new ArrayList<PipelineAggregator>(), metaData()));
        Bucket newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), new InternalAggregations(aggs));
        newBuckets.add(newBucket);
    }
    return factory.createAggregation(newBuckets);
}

Example 2 with PipelineAggregator

use of org.elasticsearch.search.aggregations.pipeline.PipelineAggregator in project elasticsearch by elastic.

the class DerivativePipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    MultiBucketsAggregation histo = (MultiBucketsAggregation) aggregation;
    List<? extends Bucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>();
    Number lastBucketKey = null;
    Double lastBucketValue = null;
    for (Bucket bucket : buckets) {
        Number thisBucketKey = factory.getKey(bucket);
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
        if (lastBucketValue != null && thisBucketValue != null) {
            double gradient = thisBucketValue - lastBucketValue;
            double xDiff = -1;
            if (xAxisUnits != null) {
                xDiff = (thisBucketKey.doubleValue() - lastBucketKey.doubleValue()) / xAxisUnits;
            }
            final List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
                return (InternalAggregation) p;
            }).collect(Collectors.toList());
            aggs.add(new InternalDerivative(name(), gradient, xDiff, formatter, new ArrayList<PipelineAggregator>(), metaData()));
            Bucket newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), new InternalAggregations(aggs));
            newBuckets.add(newBucket);
        } else {
            newBuckets.add(bucket);
        }
        lastBucketKey = thisBucketKey;
        lastBucketValue = thisBucketValue;
    }
    return factory.createAggregation(newBuckets);
}

Example 3 with PipelineAggregator

use of org.elasticsearch.search.aggregations.pipeline.PipelineAggregator in project elasticsearch by elastic.

the class MovAvgPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    MultiBucketsAggregation histo = (MultiBucketsAggregation) aggregation;
    List<? extends Bucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>();
    EvictingQueue<Double> values = new EvictingQueue<>(this.window);
    Number lastValidKey = 0;
    int lastValidPosition = 0;
    int counter = 0;
    // Do we need to fit the model parameters to the data?
    if (minimize) {
        assert (model.canBeMinimized());
        model = minimize(buckets, histo, model);
    }
    for (Bucket bucket : buckets) {
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
        // Default is to reuse existing bucket.  Simplifies the rest of the logic,
        // since we only change newBucket if we can add to it
        Bucket newBucket = bucket;
        if (!(thisBucketValue == null || thisBucketValue.equals(Double.NaN))) {
            // Some models (e.g. HoltWinters) have certain preconditions that must be met
            if (model.hasValue(values.size())) {
                double movavg = model.next(values);
                List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
                    return (InternalAggregation) p;
                }).collect(Collectors.toList());
                aggs.add(new InternalSimpleValue(name(), movavg, formatter, new ArrayList<PipelineAggregator>(), metaData()));
                newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), new InternalAggregations(aggs));
            }
            if (predict > 0) {
                lastValidKey = factory.getKey(bucket);
                lastValidPosition = counter;
            }
            values.offer(thisBucketValue);
        }
        counter += 1;
        newBuckets.add(newBucket);
    }
    if (buckets.size() > 0 && predict > 0) {
        double[] predictions = model.predict(values, predict);
        for (int i = 0; i < predictions.length; i++) {
            List<InternalAggregation> aggs;
            Number newKey = factory.nextKey(lastValidKey);
            if (lastValidPosition + i + 1 < newBuckets.size()) {
                Bucket bucket = newBuckets.get(lastValidPosition + i + 1);
                // Get the existing aggs in the bucket so we don't clobber data
                aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
                    return (InternalAggregation) p;
                }).collect(Collectors.toList());
                aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, new ArrayList<PipelineAggregator>(), metaData()));
                Bucket newBucket = factory.createBucket(newKey, 0, new InternalAggregations(aggs));
                // Overwrite the existing bucket with the new version
                newBuckets.set(lastValidPosition + i + 1, newBucket);
            } else {
                // Not seen before, create fresh
                aggs = new ArrayList<>();
                aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, new ArrayList<PipelineAggregator>(), metaData()));
                Bucket newBucket = factory.createBucket(newKey, 0, new InternalAggregations(aggs));
                // Since this is a new bucket, simply append it
                newBuckets.add(newBucket);
            }
            lastValidKey = newKey;
        }
    }
    return factory.createAggregation(newBuckets);
}

Example 4 with PipelineAggregator

use of org.elasticsearch.search.aggregations.pipeline.PipelineAggregator in project elasticsearch by elastic.

the class AggregationPhase method execute.

@Override
public void execute(SearchContext context) {
    if (context.aggregations() == null) {
        context.queryResult().aggregations(null);
        return;
    }
    if (context.queryResult().hasAggs()) {
        // no need to compute the aggs twice, they should be computed on a per context basis
        return;
    }
    Aggregator[] aggregators = context.aggregations().aggregators();
    List<Aggregator> globals = new ArrayList<>();
    for (int i = 0; i < aggregators.length; i++) {
        if (aggregators[i] instanceof GlobalAggregator) {
            globals.add(aggregators[i]);
        }
    }
    // optimize the global collector based execution
    if (!globals.isEmpty()) {
        BucketCollector globalsCollector = BucketCollector.wrap(globals);
        Query query = context.buildFilteredQuery(Queries.newMatchAllQuery());
        try {
            final Collector collector;
            if (context.getProfilers() == null) {
                collector = globalsCollector;
            } else {
                InternalProfileCollector profileCollector = new InternalProfileCollector(globalsCollector, CollectorResult.REASON_AGGREGATION_GLOBAL, // TODO: report on sub collectors
                Collections.emptyList());
                collector = profileCollector;
                // start a new profile with this collector
                context.getProfilers().addQueryProfiler().setCollector(profileCollector);
            }
            globalsCollector.preCollection();
            context.searcher().search(query, collector);
        } catch (Exception e) {
            throw new QueryPhaseExecutionException(context, "Failed to execute global aggregators", e);
        } finally {
            context.clearReleasables(SearchContext.Lifetime.COLLECTION);
        }
    }
    List<InternalAggregation> aggregations = new ArrayList<>(aggregators.length);
    for (Aggregator aggregator : context.aggregations().aggregators()) {
        try {
            aggregator.postCollection();
            aggregations.add(aggregator.buildAggregation(0));
        } catch (IOException e) {
            throw new AggregationExecutionException("Failed to build aggregation [" + aggregator.name() + "]", e);
        }
    }
    context.queryResult().aggregations(new InternalAggregations(aggregations));
    try {
        List<PipelineAggregator> pipelineAggregators = context.aggregations().factories().createPipelineAggregators();
        List<SiblingPipelineAggregator> siblingPipelineAggregators = new ArrayList<>(pipelineAggregators.size());
        for (PipelineAggregator pipelineAggregator : pipelineAggregators) {
            if (pipelineAggregator instanceof SiblingPipelineAggregator) {
                siblingPipelineAggregators.add((SiblingPipelineAggregator) pipelineAggregator);
            } else {
                throw new AggregationExecutionException("Invalid pipeline aggregation named [" + pipelineAggregator.name() + "] of type [" + pipelineAggregator.getWriteableName() + "]. Only sibling pipeline aggregations are " + "allowed at the top level");
            }
        }
        context.queryResult().pipelineAggregators(siblingPipelineAggregators);
    } catch (IOException e) {
        throw new AggregationExecutionException("Failed to build top level pipeline aggregators", e);
    }
    // disable aggregations so that they don't run on next pages in case of scrolling
    context.aggregations(null);
    context.queryCollectors().remove(AggregationPhase.class);
}

Example 5 with PipelineAggregator

use of org.elasticsearch.search.aggregations.pipeline.PipelineAggregator in project elasticsearch by elastic.

the class SerialDiffPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    MultiBucketsAggregation histo = (MultiBucketsAggregation) aggregation;
    List<? extends Bucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>();
    EvictingQueue<Double> lagWindow = new EvictingQueue<>(lag);
    int counter = 0;
    for (Bucket bucket : buckets) {
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
        Bucket newBucket = bucket;
        counter += 1;
        // Still under the initial lag period, add nothing and move on
        Double lagValue;
        if (counter <= lag) {
            lagValue = Double.NaN;
        } else {
            // Peek here, because we rely on add'ing to always move the window
            lagValue = lagWindow.peek();
        }
        // Normalize null's to NaN
        if (thisBucketValue == null) {
            thisBucketValue = Double.NaN;
        }
        // Both have values, calculate diff and replace the "empty" bucket
        if (!Double.isNaN(thisBucketValue) && !Double.isNaN(lagValue)) {
            double diff = thisBucketValue - lagValue;
            List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
                return (InternalAggregation) p;
            }).collect(Collectors.toList());
            aggs.add(new InternalSimpleValue(name(), diff, formatter, new ArrayList<PipelineAggregator>(), metaData()));
            newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), new InternalAggregations(aggs));
        }
        newBuckets.add(newBucket);
        lagWindow.add(thisBucketValue);
    }
    return factory.createAggregation(newBuckets);
}