Examples with DoublesSketchToQuantilePostAggregator org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchToQuantilePostAggregator used on opensource projects

Example 1 with DoublesSketchToQuantilePostAggregator

use of org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchToQuantilePostAggregator in project druid by druid-io.

the class DoublesSketchApproxQuantileSqlAggregator method toDruidAggregation.

@Nullable
@Override
public Aggregation toDruidAggregation(final PlannerContext plannerContext, final RowSignature rowSignature, final VirtualColumnRegistry virtualColumnRegistry, final RexBuilder rexBuilder, final String name, final AggregateCall aggregateCall, final Project project, final List<Aggregation> existingAggregations, final boolean finalizeAggregations) {
    final DruidExpression input = Aggregations.toDruidExpressionForNumericAggregator(plannerContext, rowSignature, Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(0)));
    if (input == null) {
        return null;
    }
    final AggregatorFactory aggregatorFactory;
    final String histogramName = StringUtils.format("%s:agg", name);
    final RexNode probabilityArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(1));
    if (!probabilityArg.isA(SqlKind.LITERAL)) {
        // Probability must be a literal in order to plan.
        return null;
    }
    final float probability = ((Number) RexLiteral.value(probabilityArg)).floatValue();
    final int k;
    if (aggregateCall.getArgList().size() >= 3) {
        final RexNode resolutionArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(2));
        if (!resolutionArg.isA(SqlKind.LITERAL)) {
            // Resolution must be a literal in order to plan.
            return null;
        }
        k = ((Number) RexLiteral.value(resolutionArg)).intValue();
    } else {
        k = DoublesSketchAggregatorFactory.DEFAULT_K;
    }
    // Look for existing matching aggregatorFactory.
    for (final Aggregation existing : existingAggregations) {
        for (AggregatorFactory factory : existing.getAggregatorFactories()) {
            if (factory instanceof DoublesSketchAggregatorFactory) {
                final DoublesSketchAggregatorFactory theFactory = (DoublesSketchAggregatorFactory) factory;
                // Check input for equivalence.
                final boolean inputMatches;
                final DruidExpression virtualInput = virtualColumnRegistry.findVirtualColumnExpressions(theFactory.requiredFields()).stream().findFirst().orElse(null);
                if (virtualInput == null) {
                    inputMatches = input.isDirectColumnAccess() && input.getDirectColumn().equals(theFactory.getFieldName());
                } else {
                    inputMatches = virtualInput.equals(input);
                }
                final boolean matches = inputMatches && theFactory.getK() == k;
                if (matches) {
                    // Found existing one. Use this.
                    return Aggregation.create(ImmutableList.of(), new DoublesSketchToQuantilePostAggregator(name, new FieldAccessPostAggregator(factory.getName(), factory.getName()), probability));
                }
            }
        }
    }
    // No existing match found. Create a new one.
    if (input.isDirectColumnAccess()) {
        aggregatorFactory = new DoublesSketchAggregatorFactory(histogramName, input.getDirectColumn(), k, getMaxStreamLengthFromQueryContext(plannerContext.getQueryContext()));
    } else {
        String virtualColumnName = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(input, ColumnType.FLOAT);
        aggregatorFactory = new DoublesSketchAggregatorFactory(histogramName, virtualColumnName, k, getMaxStreamLengthFromQueryContext(plannerContext.getQueryContext()));
    }
    return Aggregation.create(ImmutableList.of(aggregatorFactory), new DoublesSketchToQuantilePostAggregator(name, new FieldAccessPostAggregator(histogramName, histogramName), probability));
}

Example 2 with DoublesSketchToQuantilePostAggregator

use of org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchToQuantilePostAggregator in project druid by druid-io.

the class DoublesSketchSqlAggregatorTest method testQuantileOnCastedString.

@Test
public void testQuantileOnCastedString() throws Exception {
    final List<Object[]> expectedResults;
    if (NullHandling.replaceWithDefault()) {
        expectedResults = ImmutableList.of(new Object[] { 0.0, 1.0, 10.1, 10.1, 20.2, 0.0, 10.1, 0.0 });
    } else {
        expectedResults = ImmutableList.of(new Object[] { 1.0, 2.0, 10.1, 10.1, 20.2, Double.NaN, 10.1, Double.NaN });
    }
    testQuery("SELECT\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.01),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.5, 64),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.98, 256),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.99),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE) * 2, 0.97),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.99) FILTER(WHERE dim2 = 'abc'),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.999) FILTER(WHERE dim2 <> 'abc'),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.999) FILTER(WHERE dim2 = 'abc')\n" + "FROM foo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "CAST(\"dim1\", 'DOUBLE')", ColumnType.FLOAT, TestExprMacroTable.INSTANCE), new ExpressionVirtualColumn("v1", "(CAST(\"dim1\", 'DOUBLE') * 2)", ColumnType.FLOAT, TestExprMacroTable.INSTANCE)).aggregators(ImmutableList.of(new DoublesSketchAggregatorFactory("a0:agg", "v0", 128), new DoublesSketchAggregatorFactory("a1:agg", "v0", 64), new DoublesSketchAggregatorFactory("a2:agg", "v0", 256), new DoublesSketchAggregatorFactory("a4:agg", "v1", 128), new FilteredAggregatorFactory(new DoublesSketchAggregatorFactory("a5:agg", "v0", 128), new SelectorDimFilter("dim2", "abc", null)), new FilteredAggregatorFactory(new DoublesSketchAggregatorFactory("a6:agg", "v0", 128), new NotDimFilter(new SelectorDimFilter("dim2", "abc", null))))).postAggregators(new DoublesSketchToQuantilePostAggregator("a0", makeFieldAccessPostAgg("a0:agg"), 0.01f), new DoublesSketchToQuantilePostAggregator("a1", makeFieldAccessPostAgg("a1:agg"), 0.50f), new DoublesSketchToQuantilePostAggregator("a2", makeFieldAccessPostAgg("a2:agg"), 0.98f), new DoublesSketchToQuantilePostAggregator("a3", makeFieldAccessPostAgg("a0:agg"), 0.99f), new DoublesSketchToQuantilePostAggregator("a4", makeFieldAccessPostAgg("a4:agg"), 0.97f), new DoublesSketchToQuantilePostAggregator("a5", makeFieldAccessPostAgg("a5:agg"), 0.99f), new DoublesSketchToQuantilePostAggregator("a6", makeFieldAccessPostAgg("a6:agg"), 0.999f), new DoublesSketchToQuantilePostAggregator("a7", makeFieldAccessPostAgg("a5:agg"), 0.999f)).context(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}

Example 3 with DoublesSketchToQuantilePostAggregator

use of org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchToQuantilePostAggregator in project druid by druid-io.

the class DoublesSketchSqlAggregatorTest method testQuantileOnInnerQuery.

@Test
public void testQuantileOnInnerQuery() throws Exception {
    final List<Object[]> expectedResults;
    if (NullHandling.replaceWithDefault()) {
        expectedResults = ImmutableList.of(new Object[] { 7.0, 11.0 });
    } else {
        expectedResults = ImmutableList.of(new Object[] { 5.25, 8.0 });
    }
    testQuery("SELECT AVG(x), APPROX_QUANTILE_DS(x, 0.98)\n" + "FROM (SELECT dim2, SUM(m1) AS x FROM foo GROUP BY dim2)", Collections.singletonList(GroupByQuery.builder().setDataSource(new QueryDataSource(GroupByQuery.builder().setDataSource(CalciteTests.DATASOURCE1).setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("dim2", "d0")).setAggregatorSpecs(ImmutableList.of(new DoubleSumAggregatorFactory("a0", "m1"))).setContext(QUERY_CONTEXT_DEFAULT).build())).setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).setGranularity(Granularities.ALL).setAggregatorSpecs(new DoubleSumAggregatorFactory("_a0:sum", "a0"), new CountAggregatorFactory("_a0:count"), new DoublesSketchAggregatorFactory("_a1:agg", "a0", null)).setPostAggregatorSpecs(ImmutableList.of(new ArithmeticPostAggregator("_a0", "quotient", ImmutableList.of(new FieldAccessPostAggregator(null, "_a0:sum"), new FieldAccessPostAggregator(null, "_a0:count"))), new DoublesSketchToQuantilePostAggregator("_a1", makeFieldAccessPostAgg("_a1:agg"), 0.98f))).setContext(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}

Example 4 with DoublesSketchToQuantilePostAggregator

use of org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchToQuantilePostAggregator in project druid by druid-io.

the class DoublesSketchSqlAggregatorTest method testQuantileOnInnerQuantileQuery.

@Test
public void testQuantileOnInnerQuantileQuery() throws Exception {
    ImmutableList.Builder<Object[]> builder = ImmutableList.builder();
    builder.add(new Object[] { "", 1.0 });
    builder.add(new Object[] { "1", 4.0 });
    builder.add(new Object[] { "10.1", 2.0 });
    builder.add(new Object[] { "2", 3.0 });
    builder.add(new Object[] { "abc", 6.0 });
    builder.add(new Object[] { "def", 5.0 });
    final List<Object[]> expectedResults = builder.build();
    testQuery("SELECT dim1, APPROX_QUANTILE_DS(x, 0.5)\n" + "FROM (SELECT dim1, dim2, APPROX_QUANTILE_DS(m1, 0.5) AS x FROM foo GROUP BY dim1, dim2) GROUP BY dim1", Collections.singletonList(GroupByQuery.builder().setDataSource(new QueryDataSource(GroupByQuery.builder().setDataSource(CalciteTests.DATASOURCE1).setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("dim1", "d0"), new DefaultDimensionSpec("dim2", "d1")).setAggregatorSpecs(ImmutableList.of(new DoublesSketchAggregatorFactory("a0:agg", "m1", 128))).setPostAggregatorSpecs(ImmutableList.of(new DoublesSketchToQuantilePostAggregator("a0", makeFieldAccessPostAgg("a0:agg"), 0.5f))).setContext(QUERY_CONTEXT_DEFAULT).build())).setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("d0", "_d0", ColumnType.STRING)).setAggregatorSpecs(new DoublesSketchAggregatorFactory("_a0:agg", "a0", 128)).setPostAggregatorSpecs(ImmutableList.of(new DoublesSketchToQuantilePostAggregator("_a0", makeFieldAccessPostAgg("_a0:agg"), 0.5f))).setContext(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}

Example 5 with DoublesSketchToQuantilePostAggregator

use of org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchToQuantilePostAggregator in project druid by druid-io.

the class DoublesSketchSqlAggregatorTest method testFailWithSmallMaxStreamLength.

@Test
public void testFailWithSmallMaxStreamLength() throws Exception {
    final Map<String, Object> context = new HashMap<>(QUERY_CONTEXT_DEFAULT);
    context.put(DoublesSketchApproxQuantileSqlAggregator.CTX_APPROX_QUANTILE_DS_MAX_STREAM_LENGTH, 1);
    testQueryThrows("SELECT\n" + "APPROX_QUANTILE_DS(m1, 0.01),\n" + "APPROX_QUANTILE_DS(cnt, 0.5)\n" + "FROM foo", context, Collections.singletonList(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).aggregators(ImmutableList.of(new DoublesSketchAggregatorFactory("a0:agg", "m1", null, 1L), new DoublesSketchAggregatorFactory("a1:agg", "cnt", null, 1L))).postAggregators(new DoublesSketchToQuantilePostAggregator("a0", makeFieldAccessPostAgg("a0:agg"), 0.01f), new DoublesSketchToQuantilePostAggregator("a1", makeFieldAccessPostAgg("a1:agg"), 0.50f)).context(context).build()), expectedException -> {
        expectedException.expect(IllegalStateException.class);
        expectedException.expectMessage("NullPointerException was thrown while updating Doubles sketch");
    });
}