Examples with GapPolicy org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy used on opensource projects

Example 1 with GapPolicy

use of org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy in project elasticsearch by elastic.

the class BucketScriptPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket> originalAgg = (InternalMultiBucketAggregation<InternalMultiBucketAggregation, InternalMultiBucketAggregation.InternalBucket>) aggregation;
    List<? extends Bucket> buckets = originalAgg.getBuckets();
    CompiledScript compiledScript = reduceContext.scriptService().compile(script, ScriptContext.Standard.AGGS);
    List newBuckets = new ArrayList<>();
    for (Bucket bucket : buckets) {
        Map<String, Object> vars = new HashMap<>();
        if (script.getParams() != null) {
            vars.putAll(script.getParams());
        }
        boolean skipBucket = false;
        for (Map.Entry<String, String> entry : bucketsPathsMap.entrySet()) {
            String varName = entry.getKey();
            String bucketsPath = entry.getValue();
            Double value = resolveBucketValue(originalAgg, bucket, bucketsPath, gapPolicy);
            if (GapPolicy.SKIP == gapPolicy && (value == null || Double.isNaN(value))) {
                skipBucket = true;
                break;
            }
            vars.put(varName, value);
        }
        if (skipBucket) {
            newBuckets.add(bucket);
        } else {
            ExecutableScript executableScript = reduceContext.scriptService().executable(compiledScript, vars);
            Object returned = executableScript.run();
            if (returned == null) {
                newBuckets.add(bucket);
            } else {
                if (!(returned instanceof Number)) {
                    throw new AggregationExecutionException("series_arithmetic script for reducer [" + name() + "] must return a Number");
                }
                final List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
                    return (InternalAggregation) p;
                }).collect(Collectors.toList());
                aggs.add(new InternalSimpleValue(name(), ((Number) returned).doubleValue(), formatter, new ArrayList<>(), metaData()));
                InternalMultiBucketAggregation.InternalBucket newBucket = originalAgg.createBucket(new InternalAggregations(aggs), (InternalMultiBucketAggregation.InternalBucket) bucket);
                newBuckets.add(newBucket);
            }
        }
    }
    return originalAgg.create(newBuckets);
}

Example 2 with GapPolicy

use of org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy in project elasticsearch by elastic.

the class BucketSelectorPipelineAggregationBuilder method parse.

public static BucketSelectorPipelineAggregationBuilder parse(String reducerName, QueryParseContext context) throws IOException {
    XContentParser parser = context.parser();
    XContentParser.Token token;
    Script script = null;
    String currentFieldName = null;
    Map<String, String> bucketsPathsMap = null;
    GapPolicy gapPolicy = null;
    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
        if (token == XContentParser.Token.FIELD_NAME) {
            currentFieldName = parser.currentName();
        } else if (token == XContentParser.Token.VALUE_STRING) {
            if (BUCKETS_PATH.match(currentFieldName)) {
                bucketsPathsMap = new HashMap<>();
                bucketsPathsMap.put("_value", parser.text());
            } else if (GAP_POLICY.match(currentFieldName)) {
                gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
            } else if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
                script = Script.parse(parser);
            } else {
                throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
            }
        } else if (token == XContentParser.Token.START_ARRAY) {
            if (BUCKETS_PATH.match(currentFieldName)) {
                List<String> paths = new ArrayList<>();
                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                    String path = parser.text();
                    paths.add(path);
                }
                bucketsPathsMap = new HashMap<>();
                for (int i = 0; i < paths.size(); i++) {
                    bucketsPathsMap.put("_value" + i, paths.get(i));
                }
            } else {
                throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
            }
        } else if (token == XContentParser.Token.START_OBJECT) {
            if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName)) {
                script = Script.parse(parser);
            } else if (BUCKETS_PATH.match(currentFieldName)) {
                Map<String, Object> map = parser.map();
                bucketsPathsMap = new HashMap<>();
                for (Map.Entry<String, Object> entry : map.entrySet()) {
                    bucketsPathsMap.put(entry.getKey(), String.valueOf(entry.getValue()));
                }
            } else {
                throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
            }
        } else {
            throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " in [" + reducerName + "].");
        }
    }
    if (bucketsPathsMap == null) {
        throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + BUCKETS_PATH.getPreferredName() + "] for bucket_selector aggregation [" + reducerName + "]");
    }
    if (script == null) {
        throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + Script.SCRIPT_PARSE_FIELD.getPreferredName() + "] for bucket_selector aggregation [" + reducerName + "]");
    }
    BucketSelectorPipelineAggregationBuilder factory = new BucketSelectorPipelineAggregationBuilder(reducerName, bucketsPathsMap, script);
    if (gapPolicy != null) {
        factory.gapPolicy(gapPolicy);
    }
    return factory;
}

Example 3 with GapPolicy

use of org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy in project elasticsearch by elastic.

the class DerivativePipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    MultiBucketsAggregation histo = (MultiBucketsAggregation) aggregation;
    List<? extends Bucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>();
    Number lastBucketKey = null;
    Double lastBucketValue = null;
    for (Bucket bucket : buckets) {
        Number thisBucketKey = factory.getKey(bucket);
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
        if (lastBucketValue != null && thisBucketValue != null) {
            double gradient = thisBucketValue - lastBucketValue;
            double xDiff = -1;
            if (xAxisUnits != null) {
                xDiff = (thisBucketKey.doubleValue() - lastBucketKey.doubleValue()) / xAxisUnits;
            }
            final List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
                return (InternalAggregation) p;
            }).collect(Collectors.toList());
            aggs.add(new InternalDerivative(name(), gradient, xDiff, formatter, new ArrayList<PipelineAggregator>(), metaData()));
            Bucket newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), new InternalAggregations(aggs));
            newBuckets.add(newBucket);
        } else {
            newBuckets.add(bucket);
        }
        lastBucketKey = thisBucketKey;
        lastBucketValue = thisBucketValue;
    }
    return factory.createAggregation(newBuckets);
}

Example 4 with GapPolicy

use of org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy in project elasticsearch by elastic.

the class MovAvgPipelineAggregator method reduce.

@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
    MultiBucketsAggregation histo = (MultiBucketsAggregation) aggregation;
    List<? extends Bucket> buckets = histo.getBuckets();
    HistogramFactory factory = (HistogramFactory) histo;
    List<Bucket> newBuckets = new ArrayList<>();
    EvictingQueue<Double> values = new EvictingQueue<>(this.window);
    Number lastValidKey = 0;
    int lastValidPosition = 0;
    int counter = 0;
    // Do we need to fit the model parameters to the data?
    if (minimize) {
        assert (model.canBeMinimized());
        model = minimize(buckets, histo, model);
    }
    for (Bucket bucket : buckets) {
        Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
        // Default is to reuse existing bucket.  Simplifies the rest of the logic,
        // since we only change newBucket if we can add to it
        Bucket newBucket = bucket;
        if (!(thisBucketValue == null || thisBucketValue.equals(Double.NaN))) {
            // Some models (e.g. HoltWinters) have certain preconditions that must be met
            if (model.hasValue(values.size())) {
                double movavg = model.next(values);
                List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
                    return (InternalAggregation) p;
                }).collect(Collectors.toList());
                aggs.add(new InternalSimpleValue(name(), movavg, formatter, new ArrayList<PipelineAggregator>(), metaData()));
                newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), new InternalAggregations(aggs));
            }
            if (predict > 0) {
                lastValidKey = factory.getKey(bucket);
                lastValidPosition = counter;
            }
            values.offer(thisBucketValue);
        }
        counter += 1;
        newBuckets.add(newBucket);
    }
    if (buckets.size() > 0 && predict > 0) {
        double[] predictions = model.predict(values, predict);
        for (int i = 0; i < predictions.length; i++) {
            List<InternalAggregation> aggs;
            Number newKey = factory.nextKey(lastValidKey);
            if (lastValidPosition + i + 1 < newBuckets.size()) {
                Bucket bucket = newBuckets.get(lastValidPosition + i + 1);
                // Get the existing aggs in the bucket so we don't clobber data
                aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> {
                    return (InternalAggregation) p;
                }).collect(Collectors.toList());
                aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, new ArrayList<PipelineAggregator>(), metaData()));
                Bucket newBucket = factory.createBucket(newKey, 0, new InternalAggregations(aggs));
                // Overwrite the existing bucket with the new version
                newBuckets.set(lastValidPosition + i + 1, newBucket);
            } else {
                // Not seen before, create fresh
                aggs = new ArrayList<>();
                aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, new ArrayList<PipelineAggregator>(), metaData()));
                Bucket newBucket = factory.createBucket(newKey, 0, new InternalAggregations(aggs));
                // Since this is a new bucket, simply append it
                newBuckets.add(newBucket);
            }
            lastValidKey = newKey;
        }
    }
    return factory.createAggregation(newBuckets);
}

Example 5 with GapPolicy

use of org.elasticsearch.search.aggregations.pipeline.BucketHelpers.GapPolicy in project elasticsearch by elastic.

the class DerivativePipelineAggregationBuilder method parse.

public static DerivativePipelineAggregationBuilder parse(String pipelineAggregatorName, QueryParseContext context) throws IOException {
    XContentParser parser = context.parser();
    XContentParser.Token token;
    String currentFieldName = null;
    String[] bucketsPaths = null;
    String format = null;
    String units = null;
    GapPolicy gapPolicy = null;
    while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
        if (token == XContentParser.Token.FIELD_NAME) {
            currentFieldName = parser.currentName();
        } else if (token == XContentParser.Token.VALUE_STRING) {
            if (FORMAT_FIELD.match(currentFieldName)) {
                format = parser.text();
            } else if (BUCKETS_PATH_FIELD.match(currentFieldName)) {
                bucketsPaths = new String[] { parser.text() };
            } else if (GAP_POLICY_FIELD.match(currentFieldName)) {
                gapPolicy = GapPolicy.parse(context, parser.text(), parser.getTokenLocation());
            } else if (UNIT_FIELD.match(currentFieldName)) {
                units = parser.text();
            } else {
                throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
            }
        } else if (token == XContentParser.Token.START_ARRAY) {
            if (BUCKETS_PATH_FIELD.match(currentFieldName)) {
                List<String> paths = new ArrayList<>();
                while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
                    String path = parser.text();
                    paths.add(path);
                }
                bucketsPaths = paths.toArray(new String[paths.size()]);
            } else {
                throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
            }
        } else {
            throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " in [" + pipelineAggregatorName + "].");
        }
    }
    if (bucketsPaths == null) {
        throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + BUCKETS_PATH_FIELD.getPreferredName() + "] for derivative aggregation [" + pipelineAggregatorName + "]");
    }
    DerivativePipelineAggregationBuilder factory = new DerivativePipelineAggregationBuilder(pipelineAggregatorName, bucketsPaths[0]);
    if (format != null) {
        factory.format(format);
    }
    if (gapPolicy != null) {
        factory.gapPolicy(gapPolicy);
    }
    if (units != null) {
        factory.unit(units);
    }
    return factory;
}