use of org.apache.druid.query.aggregation.histogram.FixedBucketsHistogramAggregatorFactory in project druid by druid-io.
the class FixedBucketsHistogramQuantileSqlAggregator method toDruidAggregation.
@Nullable
@Override
public Aggregation toDruidAggregation(PlannerContext plannerContext, RowSignature rowSignature, VirtualColumnRegistry virtualColumnRegistry, RexBuilder rexBuilder, String name, AggregateCall aggregateCall, Project project, List<Aggregation> existingAggregations, boolean finalizeAggregations) {
final DruidExpression input = Aggregations.toDruidExpressionForNumericAggregator(plannerContext, rowSignature, Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(0)));
if (input == null) {
return null;
}
final AggregatorFactory aggregatorFactory;
final String histogramName = StringUtils.format("%s:agg", name);
final RexNode probabilityArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(1));
if (!probabilityArg.isA(SqlKind.LITERAL)) {
// Probability must be a literal in order to plan.
return null;
}
final float probability = ((Number) RexLiteral.value(probabilityArg)).floatValue();
final int numBuckets;
if (aggregateCall.getArgList().size() >= 3) {
final RexNode numBucketsArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(2));
if (!numBucketsArg.isA(SqlKind.LITERAL)) {
// Resolution must be a literal in order to plan.
return null;
}
numBuckets = ((Number) RexLiteral.value(numBucketsArg)).intValue();
} else {
return null;
}
final double lowerLimit;
if (aggregateCall.getArgList().size() >= 4) {
final RexNode lowerLimitArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(3));
if (!lowerLimitArg.isA(SqlKind.LITERAL)) {
// Resolution must be a literal in order to plan.
return null;
}
lowerLimit = ((Number) RexLiteral.value(lowerLimitArg)).doubleValue();
} else {
return null;
}
final double upperLimit;
if (aggregateCall.getArgList().size() >= 5) {
final RexNode upperLimitArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(4));
if (!upperLimitArg.isA(SqlKind.LITERAL)) {
// Resolution must be a literal in order to plan.
return null;
}
upperLimit = ((Number) RexLiteral.value(upperLimitArg)).doubleValue();
} else {
return null;
}
final FixedBucketsHistogram.OutlierHandlingMode outlierHandlingMode;
if (aggregateCall.getArgList().size() >= 6) {
final RexNode outlierHandlingModeArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(5));
if (!outlierHandlingModeArg.isA(SqlKind.LITERAL)) {
// Resolution must be a literal in order to plan.
return null;
}
outlierHandlingMode = FixedBucketsHistogram.OutlierHandlingMode.fromString(RexLiteral.stringValue(outlierHandlingModeArg));
} else {
outlierHandlingMode = FixedBucketsHistogram.OutlierHandlingMode.IGNORE;
}
// Look for existing matching aggregatorFactory.
for (final Aggregation existing : existingAggregations) {
for (AggregatorFactory factory : existing.getAggregatorFactories()) {
if (factory instanceof FixedBucketsHistogramAggregatorFactory) {
final FixedBucketsHistogramAggregatorFactory theFactory = (FixedBucketsHistogramAggregatorFactory) factory;
// Check input for equivalence.
final boolean inputMatches;
final DruidExpression virtualInput = virtualColumnRegistry.findVirtualColumnExpressions(theFactory.requiredFields()).stream().findFirst().orElse(null);
if (virtualInput == null) {
inputMatches = input.isDirectColumnAccess() && input.getDirectColumn().equals(theFactory.getFieldName());
} else {
inputMatches = virtualInput.equals(input);
}
final boolean matches = inputMatches && theFactory.getOutlierHandlingMode() == outlierHandlingMode && theFactory.getNumBuckets() == numBuckets && theFactory.getLowerLimit() == lowerLimit && theFactory.getUpperLimit() == upperLimit;
if (matches) {
// Found existing one. Use this.
return Aggregation.create(ImmutableList.of(), new QuantilePostAggregator(name, factory.getName(), probability));
}
}
}
}
// No existing match found. Create a new one.
if (input.isDirectColumnAccess()) {
aggregatorFactory = new FixedBucketsHistogramAggregatorFactory(histogramName, input.getDirectColumn(), numBuckets, lowerLimit, upperLimit, outlierHandlingMode, false);
} else {
String virtualColumnName = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(input, ColumnType.FLOAT);
aggregatorFactory = new FixedBucketsHistogramAggregatorFactory(histogramName, virtualColumnName, numBuckets, lowerLimit, upperLimit, outlierHandlingMode, false);
}
return Aggregation.create(ImmutableList.of(aggregatorFactory), new QuantilePostAggregator(name, histogramName, probability));
}
use of org.apache.druid.query.aggregation.histogram.FixedBucketsHistogramAggregatorFactory in project druid by druid-io.
the class FixedBucketsHistogramQuantileSqlAggregatorTest method testQuantileOnInnerQuery.
@Test
public void testQuantileOnInnerQuery() throws Exception {
final List<Object[]> expectedResults;
if (NullHandling.replaceWithDefault()) {
expectedResults = ImmutableList.of(new Object[] { 7.0, 11.940000534057617 });
} else {
expectedResults = ImmutableList.of(new Object[] { 5.25, 8.920000076293945 });
}
testQuery("SELECT AVG(x), APPROX_QUANTILE_FIXED_BUCKETS(x, 0.98, 100, 0.0, 100.0)\n" + "FROM (SELECT dim2, SUM(m1) AS x FROM foo GROUP BY dim2)", ImmutableList.of(GroupByQuery.builder().setDataSource(new QueryDataSource(GroupByQuery.builder().setDataSource(CalciteTests.DATASOURCE1).setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("dim2", "d0")).setAggregatorSpecs(ImmutableList.of(new DoubleSumAggregatorFactory("a0", "m1"))).setContext(QUERY_CONTEXT_DEFAULT).build())).setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).setGranularity(Granularities.ALL).setAggregatorSpecs(new DoubleSumAggregatorFactory("_a0:sum", "a0"), new CountAggregatorFactory("_a0:count"), new FixedBucketsHistogramAggregatorFactory("_a1:agg", "a0", 100, 0, 100.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false)).setPostAggregatorSpecs(ImmutableList.of(new ArithmeticPostAggregator("_a0", "quotient", ImmutableList.of(new FieldAccessPostAggregator(null, "_a0:sum"), new FieldAccessPostAggregator(null, "_a0:count"))), new QuantilePostAggregator("_a1", "_a1:agg", 0.98f))).setContext(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}
use of org.apache.druid.query.aggregation.histogram.FixedBucketsHistogramAggregatorFactory in project druid by druid-io.
the class FixedBucketsHistogramQuantileSqlAggregatorTest method testQuantileOnFloatAndLongs.
@Test
public void testQuantileOnFloatAndLongs() throws Exception {
final List<Object[]> expectedResults = ImmutableList.of(new Object[] { 1.0299999713897705, 3.5, 6.440000057220459, 6.470000267028809, 12.40999984741211, 6.494999885559082, 5.497499942779541, 6.499499797821045, 1.25 });
testQuery("SELECT\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.01, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.5, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.98, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.99, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1 * 2, 0.97, 40, 0.0, 20.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.99, 20, 0.0, 10.0) FILTER(WHERE dim1 = 'abc'),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.999, 20, 0.0, 10.0) FILTER(WHERE dim1 <> 'abc'),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.999, 20, 0.0, 10.0) FILTER(WHERE dim1 = 'abc'),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(cnt, 0.5, 20, 0.0, 10.0)\n" + "FROM foo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "(\"m1\" * 2)", ColumnType.FLOAT, TestExprMacroTable.INSTANCE)).aggregators(ImmutableList.of(new FixedBucketsHistogramAggregatorFactory("a0:agg", "m1", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new FixedBucketsHistogramAggregatorFactory("a4:agg", "v0", 40, 0.0d, 20.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new FilteredAggregatorFactory(new FixedBucketsHistogramAggregatorFactory("a5:agg", "m1", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new SelectorDimFilter("dim1", "abc", null)), new FilteredAggregatorFactory(new FixedBucketsHistogramAggregatorFactory("a6:agg", "m1", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))), new FixedBucketsHistogramAggregatorFactory("a8:agg", "cnt", 20, 0.0d, 10.0d, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false))).postAggregators(new QuantilePostAggregator("a0", "a0:agg", 0.01f), new QuantilePostAggregator("a1", "a0:agg", 0.50f), new QuantilePostAggregator("a2", "a0:agg", 0.98f), new QuantilePostAggregator("a3", "a0:agg", 0.99f), new QuantilePostAggregator("a4", "a4:agg", 0.97f), new QuantilePostAggregator("a5", "a5:agg", 0.99f), new QuantilePostAggregator("a6", "a6:agg", 0.999f), new QuantilePostAggregator("a7", "a5:agg", 0.999f), new QuantilePostAggregator("a8", "a8:agg", 0.50f)).context(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}
use of org.apache.druid.query.aggregation.histogram.FixedBucketsHistogramAggregatorFactory in project druid by druid-io.
the class FixedBucketsHistogramQuantileSqlAggregatorTest method testEmptyTimeseriesResults.
@Test
public void testEmptyTimeseriesResults() throws Exception {
cannotVectorize();
testQuery("SELECT\n" + "APPROX_QUANTILE_FIXED_BUCKETS(fbhist_m1, 0.01, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(m1, 0.01, 20, 0.0, 10.0)\n" + "FROM foo WHERE dim2 = 0", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).filters(bound("dim2", "0", "0", false, false, null, StringComparators.NUMERIC)).aggregators(ImmutableList.of(new FixedBucketsHistogramAggregatorFactory("a0:agg", "fbhist_m1", 20, 0.0, 10.0, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new FixedBucketsHistogramAggregatorFactory("a1:agg", "m1", 20, 0.0, 10.0, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false))).postAggregators(new QuantilePostAggregator("a0", "a0:agg", 0.01f), new QuantilePostAggregator("a1", "a1:agg", 0.01f)).context(QUERY_CONTEXT_DEFAULT).build()), ImmutableList.of(new Object[] { 0.0, 0.0 }));
}
use of org.apache.druid.query.aggregation.histogram.FixedBucketsHistogramAggregatorFactory in project druid by druid-io.
the class FixedBucketsHistogramQuantileSqlAggregatorTest method testQuantileOnComplexColumn.
@Test
public void testQuantileOnComplexColumn() throws Exception {
cannotVectorize();
final List<Object[]> expectedResults = ImmutableList.of(new Object[] { 1.0299999713897705, 3.5, 6.293333530426025, 6.470000267028809, 6.494999885559082, 5.497499942779541, 6.499499797821045 });
testQuery("SELECT\n" + "APPROX_QUANTILE_FIXED_BUCKETS(fbhist_m1, 0.01, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(fbhist_m1, 0.5, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(fbhist_m1, 0.98, 30, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(fbhist_m1, 0.99, 20, 0.0, 10.0),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(fbhist_m1, 0.99, 20, 0.0, 10.0) FILTER(WHERE dim1 = 'abc'),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(fbhist_m1, 0.999, 20, 0.0, 10.0) FILTER(WHERE dim1 <> 'abc'),\n" + "APPROX_QUANTILE_FIXED_BUCKETS(fbhist_m1, 0.999, 20, 0.0, 10.0) FILTER(WHERE dim1 = 'abc')\n" + "FROM foo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).aggregators(ImmutableList.of(new FixedBucketsHistogramAggregatorFactory("a0:agg", "fbhist_m1", 20, 0.0, 10.0, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new FixedBucketsHistogramAggregatorFactory("a2:agg", "fbhist_m1", 30, 0.0, 10.0, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new FilteredAggregatorFactory(new FixedBucketsHistogramAggregatorFactory("a4:agg", "fbhist_m1", 20, 0.0, 10.0, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new SelectorDimFilter("dim1", "abc", null)), new FilteredAggregatorFactory(new FixedBucketsHistogramAggregatorFactory("a5:agg", "fbhist_m1", 20, 0.0, 10.0, FixedBucketsHistogram.OutlierHandlingMode.IGNORE, false), new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))))).postAggregators(new QuantilePostAggregator("a0", "a0:agg", 0.01f), new QuantilePostAggregator("a1", "a0:agg", 0.50f), new QuantilePostAggregator("a2", "a2:agg", 0.98f), new QuantilePostAggregator("a3", "a0:agg", 0.99f), new QuantilePostAggregator("a4", "a4:agg", 0.99f), new QuantilePostAggregator("a5", "a5:agg", 0.999f), new QuantilePostAggregator("a6", "a4:agg", 0.999f)).context(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}
Aggregations