use of org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy in project OpenSearch by opensearch-project.
the class BucketMetricsParser method parse.
@Override
public final BucketMetricsPipelineAggregationBuilder<?> parse(String pipelineAggregatorName, XContentParser parser) throws IOException {
XContentParser.Token token;
String currentFieldName = null;
String[] bucketsPaths = null;
String format = null;
GapPolicy gapPolicy = null;
Map<String, Object> params = new HashMap<>(5);
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_STRING) {
if (FORMAT.match(currentFieldName, parser.getDeprecationHandler())) {
format = parser.text();
} else if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) {
bucketsPaths = new String[] { parser.text() };
} else if (GAP_POLICY.match(currentFieldName, parser.getDeprecationHandler())) {
gapPolicy = GapPolicy.parse(parser.text(), parser.getTokenLocation());
} else {
parseToken(pipelineAggregatorName, parser, currentFieldName, token, params);
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) {
List<String> paths = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
String path = parser.text();
paths.add(path);
}
bucketsPaths = paths.toArray(new String[paths.size()]);
} else {
parseToken(pipelineAggregatorName, parser, currentFieldName, token, params);
}
} else {
parseToken(pipelineAggregatorName, parser, currentFieldName, token, params);
}
}
if (bucketsPaths == null) {
throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + BUCKETS_PATH.getPreferredName() + "] for aggregation [" + pipelineAggregatorName + "]");
}
BucketMetricsPipelineAggregationBuilder<?> factory = buildFactory(pipelineAggregatorName, bucketsPaths[0], params);
if (format != null) {
factory.format(format);
}
if (gapPolicy != null) {
factory.gapPolicy(gapPolicy);
}
assert (factory != null);
return factory;
}
use of org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy in project OpenSearch by opensearch-project.
the class MovAvgPipelineAggregator method reduce.
@Override
public InternalAggregation reduce(InternalAggregation aggregation, ReduceContext reduceContext) {
InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket> histo = (InternalMultiBucketAggregation<? extends InternalMultiBucketAggregation, ? extends InternalMultiBucketAggregation.InternalBucket>) aggregation;
List<? extends InternalMultiBucketAggregation.InternalBucket> buckets = histo.getBuckets();
HistogramFactory factory = (HistogramFactory) histo;
List<Bucket> newBuckets = new ArrayList<>();
EvictingQueue<Double> values = new EvictingQueue<>(this.window);
Number lastValidKey = 0;
int lastValidPosition = 0;
int counter = 0;
// Do we need to fit the model parameters to the data?
if (minimize) {
assert (model.canBeMinimized());
model = minimize(buckets, histo, model);
}
for (InternalMultiBucketAggregation.InternalBucket bucket : buckets) {
Double thisBucketValue = resolveBucketValue(histo, bucket, bucketsPaths()[0], gapPolicy);
// Default is to reuse existing bucket. Simplifies the rest of the logic,
// since we only change newBucket if we can add to it
Bucket newBucket = bucket;
if ((thisBucketValue == null || thisBucketValue.equals(Double.NaN)) == false) {
// Some models (e.g. HoltWinters) have certain preconditions that must be met
if (model.hasValue(values.size())) {
double movavg = model.next(values);
List<InternalAggregation> aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> (InternalAggregation) p).collect(Collectors.toList());
aggs.add(new InternalSimpleValue(name(), movavg, formatter, metadata()));
newBucket = factory.createBucket(factory.getKey(bucket), bucket.getDocCount(), InternalAggregations.from(aggs));
}
if (predict > 0) {
lastValidKey = factory.getKey(bucket);
lastValidPosition = counter;
}
values.offer(thisBucketValue);
}
counter += 1;
newBuckets.add(newBucket);
}
if (buckets.size() > 0 && predict > 0) {
double[] predictions = model.predict(values, predict);
for (int i = 0; i < predictions.length; i++) {
List<InternalAggregation> aggs;
Number newKey = factory.nextKey(lastValidKey);
if (lastValidPosition + i + 1 < newBuckets.size()) {
Bucket bucket = newBuckets.get(lastValidPosition + i + 1);
// Get the existing aggs in the bucket so we don't clobber data
aggs = StreamSupport.stream(bucket.getAggregations().spliterator(), false).map((p) -> (InternalAggregation) p).collect(Collectors.toList());
aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, metadata()));
Bucket newBucket = factory.createBucket(newKey, bucket.getDocCount(), InternalAggregations.from(aggs));
// Overwrite the existing bucket with the new version
newBuckets.set(lastValidPosition + i + 1, newBucket);
} else {
// Not seen before, create fresh
aggs = new ArrayList<>();
aggs.add(new InternalSimpleValue(name(), predictions[i], formatter, metadata()));
Bucket newBucket = factory.createBucket(newKey, 0, InternalAggregations.from(aggs));
// Since this is a new bucket, simply append it
newBuckets.add(newBucket);
}
lastValidKey = newKey;
}
}
return factory.createAggregation(newBuckets);
}
use of org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy in project OpenSearch by opensearch-project.
the class BucketSelectorPipelineAggregationBuilder method parse.
public static BucketSelectorPipelineAggregationBuilder parse(String reducerName, XContentParser parser) throws IOException {
XContentParser.Token token;
Script script = null;
String currentFieldName = null;
Map<String, String> bucketsPathsMap = null;
GapPolicy gapPolicy = null;
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_STRING) {
if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) {
bucketsPathsMap = new HashMap<>();
bucketsPathsMap.put("_value", parser.text());
} else if (GAP_POLICY.match(currentFieldName, parser.getDeprecationHandler())) {
gapPolicy = GapPolicy.parse(parser.text(), parser.getTokenLocation());
} else if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
script = Script.parse(parser);
} else {
throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) {
List<String> paths = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
String path = parser.text();
paths.add(path);
}
bucketsPathsMap = new HashMap<>();
for (int i = 0; i < paths.size(); i++) {
bucketsPathsMap.put("_value" + i, paths.get(i));
}
} else {
throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
}
} else if (token == XContentParser.Token.START_OBJECT) {
if (Script.SCRIPT_PARSE_FIELD.match(currentFieldName, parser.getDeprecationHandler())) {
script = Script.parse(parser);
} else if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) {
Map<String, Object> map = parser.map();
bucketsPathsMap = new HashMap<>();
for (Map.Entry<String, Object> entry : map.entrySet()) {
bucketsPathsMap.put(entry.getKey(), String.valueOf(entry.getValue()));
}
} else {
throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + reducerName + "]: [" + currentFieldName + "].");
}
} else {
throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " in [" + reducerName + "].");
}
}
if (bucketsPathsMap == null) {
throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + BUCKETS_PATH.getPreferredName() + "] for bucket_selector aggregation [" + reducerName + "]");
}
if (script == null) {
throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + Script.SCRIPT_PARSE_FIELD.getPreferredName() + "] for bucket_selector aggregation [" + reducerName + "]");
}
BucketSelectorPipelineAggregationBuilder factory = new BucketSelectorPipelineAggregationBuilder(reducerName, bucketsPathsMap, script);
if (gapPolicy != null) {
factory.gapPolicy(gapPolicy);
}
return factory;
}
use of org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy in project OpenSearch by opensearch-project.
the class MovAvgPipelineAggregationBuilder method parse.
public static MovAvgPipelineAggregationBuilder parse(ParseFieldRegistry<MovAvgModel.AbstractModelParser> movingAverageMdelParserRegistry, String pipelineAggregatorName, XContentParser parser) throws IOException {
XContentParser.Token token;
String currentFieldName = null;
String[] bucketsPaths = null;
String format = null;
GapPolicy gapPolicy = null;
Integer window = null;
Map<String, Object> settings = null;
String model = null;
Integer predict = null;
Boolean minimize = null;
DEPRECATION_LOGGER.deprecate("moving_avg_aggregation", "The moving_avg aggregation has been deprecated in favor of the moving_fn aggregation.");
while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) {
if (token == XContentParser.Token.FIELD_NAME) {
currentFieldName = parser.currentName();
} else if (token == XContentParser.Token.VALUE_NUMBER) {
if (WINDOW.match(currentFieldName, parser.getDeprecationHandler())) {
window = parser.intValue();
if (window <= 0) {
throw new ParsingException(parser.getTokenLocation(), "[" + currentFieldName + "] value must be a positive, " + "non-zero integer. Value supplied was [" + predict + "] in [" + pipelineAggregatorName + "].");
}
} else if (PREDICT.match(currentFieldName, parser.getDeprecationHandler())) {
predict = parser.intValue();
if (predict <= 0) {
throw new ParsingException(parser.getTokenLocation(), "[" + currentFieldName + "] value must be a positive integer." + " Value supplied was [" + predict + "] in [" + pipelineAggregatorName + "].");
}
} else {
throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
}
} else if (token == XContentParser.Token.VALUE_STRING) {
if (FORMAT.match(currentFieldName, parser.getDeprecationHandler())) {
format = parser.text();
} else if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) {
bucketsPaths = new String[] { parser.text() };
} else if (GAP_POLICY.match(currentFieldName, parser.getDeprecationHandler())) {
gapPolicy = GapPolicy.parse(parser.text(), parser.getTokenLocation());
} else if (MODEL.match(currentFieldName, parser.getDeprecationHandler())) {
model = parser.text();
} else {
throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
}
} else if (token == XContentParser.Token.START_ARRAY) {
if (BUCKETS_PATH.match(currentFieldName, parser.getDeprecationHandler())) {
List<String> paths = new ArrayList<>();
while ((token = parser.nextToken()) != XContentParser.Token.END_ARRAY) {
String path = parser.text();
paths.add(path);
}
bucketsPaths = paths.toArray(new String[paths.size()]);
} else {
throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
}
} else if (token == XContentParser.Token.START_OBJECT) {
if (SETTINGS.match(currentFieldName, parser.getDeprecationHandler())) {
settings = parser.map();
} else {
throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
}
} else if (token == XContentParser.Token.VALUE_BOOLEAN) {
if (MINIMIZE.match(currentFieldName, parser.getDeprecationHandler())) {
minimize = parser.booleanValue();
} else {
throw new ParsingException(parser.getTokenLocation(), "Unknown key for a " + token + " in [" + pipelineAggregatorName + "]: [" + currentFieldName + "].");
}
} else {
throw new ParsingException(parser.getTokenLocation(), "Unexpected token " + token + " in [" + pipelineAggregatorName + "].");
}
}
if (bucketsPaths == null) {
throw new ParsingException(parser.getTokenLocation(), "Missing required field [" + BUCKETS_PATH.getPreferredName() + "] for movingAvg aggregation [" + pipelineAggregatorName + "]");
}
MovAvgPipelineAggregationBuilder factory = new MovAvgPipelineAggregationBuilder(pipelineAggregatorName, bucketsPaths[0]);
if (format != null) {
factory.format(format);
}
if (gapPolicy != null) {
factory.gapPolicy(gapPolicy);
}
if (window != null) {
factory.window(window);
}
if (predict != null) {
factory.predict(predict);
}
if (model != null) {
MovAvgModel.AbstractModelParser modelParser = movingAverageMdelParserRegistry.lookup(model, parser.getTokenLocation(), parser.getDeprecationHandler());
MovAvgModel movAvgModel;
try {
movAvgModel = modelParser.parse(settings, pipelineAggregatorName, factory.window());
} catch (ParseException exception) {
throw new ParsingException(parser.getTokenLocation(), "Could not parse settings for model [" + model + "].", exception);
}
factory.model(movAvgModel);
}
if (minimize != null) {
factory.minimize(minimize);
}
return factory;
}
use of org.opensearch.search.aggregations.pipeline.BucketHelpers.GapPolicy in project OpenSearch by opensearch-project.
the class DerivativeIT method testSingleValueAggDerivativeWithGaps_random.
public void testSingleValueAggDerivativeWithGaps_random() throws Exception {
GapPolicy gapPolicy = randomFrom(GapPolicy.values());
SearchResponse searchResponse = client().prepareSearch("empty_bucket_idx_rnd").setQuery(matchAllQuery()).addAggregation(histogram("histo").field(SINGLE_VALUED_FIELD_NAME).interval(1).extendedBounds(0L, (long) numBuckets_empty_rnd - 1).subAggregation(sum("sum").field(SINGLE_VALUED_FIELD_NAME)).subAggregation(derivative("deriv", "sum").gapPolicy(gapPolicy))).get();
assertThat(searchResponse.getHits().getTotalHits().value, equalTo(numDocsEmptyIdx_rnd));
Histogram deriv = searchResponse.getAggregations().get("histo");
assertThat(deriv, Matchers.notNullValue());
assertThat(deriv.getName(), equalTo("histo"));
List<? extends Bucket> buckets = deriv.getBuckets();
assertThat(buckets.size(), equalTo(numBuckets_empty_rnd));
double lastSumValue = Double.NaN;
for (int i = 0; i < valueCounts_empty_rnd.length; i++) {
Histogram.Bucket bucket = buckets.get(i);
checkBucketKeyAndDocCount("InternalBucket " + i, bucket, i, valueCounts_empty_rnd[i]);
Sum sum = bucket.getAggregations().get("sum");
double thisSumValue = sum.value();
if (bucket.getDocCount() == 0) {
thisSumValue = gapPolicy == GapPolicy.INSERT_ZEROS ? 0 : Double.NaN;
}
SimpleValue sumDeriv = bucket.getAggregations().get("deriv");
if (i == 0) {
assertThat(sumDeriv, nullValue());
} else {
double expectedDerivative = thisSumValue - lastSumValue;
if (Double.isNaN(expectedDerivative)) {
assertThat(sumDeriv.value(), equalTo(expectedDerivative));
} else {
assertThat(sumDeriv.value(), closeTo(expectedDerivative, 0.00001));
}
}
lastSumValue = thisSumValue;
}
}
Aggregations