Examples with QuantilePostAggregator io.druid.query.aggregation.histogram.QuantilePostAggregator used on opensource projects

Example 1 with QuantilePostAggregator

use of io.druid.query.aggregation.histogram.QuantilePostAggregator in project druid by druid-io.

the class QuantileSqlAggregatorTest method testQuantileOnComplexColumn.

@Test
public void testQuantileOnComplexColumn() throws Exception {
    try (final DruidPlanner planner = plannerFactory.createPlanner(null)) {
        final String sql = "SELECT\n" + "APPROX_QUANTILE(hist_m1, 0.01),\n" + "APPROX_QUANTILE(hist_m1, 0.5, 50),\n" + "APPROX_QUANTILE(hist_m1, 0.98, 200),\n" + "APPROX_QUANTILE(hist_m1, 0.99),\n" + "APPROX_QUANTILE(hist_m1, 0.99) FILTER(WHERE dim1 = 'abc'),\n" + "APPROX_QUANTILE(hist_m1, 0.999) FILTER(WHERE dim1 <> 'abc'),\n" + "APPROX_QUANTILE(hist_m1, 0.999) FILTER(WHERE dim1 = 'abc')\n" + "FROM foo";
        final PlannerResult plannerResult = planner.plan(sql);
        // Verify results
        final List<Object[]> results = Sequences.toList(plannerResult.run(), new ArrayList<Object[]>());
        final List<Object[]> expectedResults = ImmutableList.of(new Object[] { 1.0, 3.0, 5.880000114440918, 5.940000057220459, 6.0, 4.994999885559082, 6.0 });
        Assert.assertEquals(expectedResults.size(), results.size());
        for (int i = 0; i < expectedResults.size(); i++) {
            Assert.assertArrayEquals(expectedResults.get(i), results.get(i));
        }
        // Verify query
        Assert.assertEquals(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).aggregators(ImmutableList.of(new ApproximateHistogramFoldingAggregatorFactory("a0:agg", "hist_m1", null, null, null, null), new ApproximateHistogramFoldingAggregatorFactory("a2:agg", "hist_m1", 200, null, null, null), new FilteredAggregatorFactory(new ApproximateHistogramFoldingAggregatorFactory("a4:agg", "hist_m1", null, null, null, null), new SelectorDimFilter("dim1", "abc", null)), new FilteredAggregatorFactory(new ApproximateHistogramFoldingAggregatorFactory("a5:agg", "hist_m1", null, null, null, null), new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))))).postAggregators(ImmutableList.<PostAggregator>of(new QuantilePostAggregator("a0", "a0:agg", 0.01f), new QuantilePostAggregator("a1", "a0:agg", 0.50f), new QuantilePostAggregator("a2", "a2:agg", 0.98f), new QuantilePostAggregator("a3", "a0:agg", 0.99f), new QuantilePostAggregator("a4", "a4:agg", 0.99f), new QuantilePostAggregator("a5", "a5:agg", 0.999f), new QuantilePostAggregator("a6", "a4:agg", 0.999f))).context(ImmutableMap.<String, Object>of("skipEmptyBuckets", true)).build(), Iterables.getOnlyElement(queryLogHook.getRecordedQueries()));
    }
}

Example 2 with QuantilePostAggregator

use of io.druid.query.aggregation.histogram.QuantilePostAggregator in project druid by druid-io.

the class QuantileSqlAggregator method toDruidAggregation.

@Override
public Aggregation toDruidAggregation(final String name, final RowSignature rowSignature, final DruidOperatorTable operatorTable, final PlannerContext plannerContext, final List<Aggregation> existingAggregations, final Project project, final AggregateCall aggregateCall, final DimFilter filter) {
    final RowExtraction rex = Expressions.toRowExtraction(operatorTable, plannerContext, rowSignature.getRowOrder(), Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(0)));
    if (rex == null) {
        return null;
    }
    final AggregatorFactory aggregatorFactory;
    final String histogramName = String.format("%s:agg", name);
    final RexNode probabilityArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(1));
    final float probability = ((Number) RexLiteral.value(probabilityArg)).floatValue();
    final int resolution;
    if (aggregateCall.getArgList().size() >= 3) {
        final RexNode resolutionArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(2));
        resolution = ((Number) RexLiteral.value(resolutionArg)).intValue();
    } else {
        resolution = ApproximateHistogram.DEFAULT_HISTOGRAM_SIZE;
    }
    final int numBuckets = ApproximateHistogram.DEFAULT_BUCKET_SIZE;
    final float lowerLimit = Float.NEGATIVE_INFINITY;
    final float upperLimit = Float.POSITIVE_INFINITY;
    // Look for existing matching aggregatorFactory.
    for (final Aggregation existing : existingAggregations) {
        for (AggregatorFactory factory : existing.getAggregatorFactories()) {
            final boolean matches = Aggregations.aggregatorMatches(factory, filter, ApproximateHistogramAggregatorFactory.class, new Predicate<ApproximateHistogramAggregatorFactory>() {

                @Override
                public boolean apply(final ApproximateHistogramAggregatorFactory theFactory) {
                    return theFactory.getFieldName().equals(rex.getColumn()) && theFactory.getResolution() == resolution && theFactory.getNumBuckets() == numBuckets && theFactory.getLowerLimit() == lowerLimit && theFactory.getUpperLimit() == upperLimit;
                }
            });
            if (matches) {
                // Found existing one. Use this.
                return Aggregation.create(ImmutableList.<AggregatorFactory>of(), new QuantilePostAggregator(name, factory.getName(), probability));
            }
        }
    }
    if (rowSignature.getColumnType(rex.getColumn()) == ValueType.COMPLEX) {
        aggregatorFactory = new ApproximateHistogramFoldingAggregatorFactory(histogramName, rex.getColumn(), resolution, numBuckets, lowerLimit, upperLimit);
    } else {
        aggregatorFactory = new ApproximateHistogramAggregatorFactory(histogramName, rex.getColumn(), resolution, numBuckets, lowerLimit, upperLimit);
    }
    return Aggregation.create(ImmutableList.of(aggregatorFactory), new QuantilePostAggregator(name, histogramName, probability)).filter(filter);
}

Example 3 with QuantilePostAggregator

use of io.druid.query.aggregation.histogram.QuantilePostAggregator in project druid by druid-io.

the class QuantileSqlAggregatorTest method testQuantileOnFloatAndLongs.

@Test
public void testQuantileOnFloatAndLongs() throws Exception {
    try (final DruidPlanner planner = plannerFactory.createPlanner(null)) {
        final String sql = "SELECT\n" + "APPROX_QUANTILE(m1, 0.01),\n" + "APPROX_QUANTILE(m1, 0.5, 50),\n" + "APPROX_QUANTILE(m1, 0.98, 200),\n" + "APPROX_QUANTILE(m1, 0.99),\n" + "APPROX_QUANTILE(m1, 0.99) FILTER(WHERE dim1 = 'abc'),\n" + "APPROX_QUANTILE(m1, 0.999) FILTER(WHERE dim1 <> 'abc'),\n" + "APPROX_QUANTILE(m1, 0.999) FILTER(WHERE dim1 = 'abc'),\n" + "APPROX_QUANTILE(cnt, 0.5)\n" + "FROM foo";
        final PlannerResult plannerResult = planner.plan(sql);
        // Verify results
        final List<Object[]> results = Sequences.toList(plannerResult.run(), new ArrayList<Object[]>());
        final List<Object[]> expectedResults = ImmutableList.of(new Object[] { 1.0, 3.0, 5.880000114440918, 5.940000057220459, 6.0, 4.994999885559082, 6.0, 1.0 });
        Assert.assertEquals(expectedResults.size(), results.size());
        for (int i = 0; i < expectedResults.size(); i++) {
            Assert.assertArrayEquals(expectedResults.get(i), results.get(i));
        }
        // Verify query
        Assert.assertEquals(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).aggregators(ImmutableList.of(new ApproximateHistogramAggregatorFactory("a0:agg", "m1", null, null, null, null), new ApproximateHistogramAggregatorFactory("a2:agg", "m1", 200, null, null, null), new FilteredAggregatorFactory(new ApproximateHistogramAggregatorFactory("a4:agg", "m1", null, null, null, null), new SelectorDimFilter("dim1", "abc", null)), new FilteredAggregatorFactory(new ApproximateHistogramAggregatorFactory("a5:agg", "m1", null, null, null, null), new NotDimFilter(new SelectorDimFilter("dim1", "abc", null))), new ApproximateHistogramAggregatorFactory("a7:agg", "cnt", null, null, null, null))).postAggregators(ImmutableList.<PostAggregator>of(new QuantilePostAggregator("a0", "a0:agg", 0.01f), new QuantilePostAggregator("a1", "a0:agg", 0.50f), new QuantilePostAggregator("a2", "a2:agg", 0.98f), new QuantilePostAggregator("a3", "a0:agg", 0.99f), new QuantilePostAggregator("a4", "a4:agg", 0.99f), new QuantilePostAggregator("a5", "a5:agg", 0.999f), new QuantilePostAggregator("a6", "a4:agg", 0.999f), new QuantilePostAggregator("a7", "a7:agg", 0.50f))).context(ImmutableMap.<String, Object>of("skipEmptyBuckets", true)).build(), Iterables.getOnlyElement(queryLogHook.getRecordedQueries()));
    }
}