Examples with TDigestSketchToQuantilePostAggregator org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchToQuantilePostAggregator used on opensource projects

Example 1 with TDigestSketchToQuantilePostAggregator

use of org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchToQuantilePostAggregator in project druid by druid-io.

the class TDigestSketchQuantileSqlAggregator method toDruidAggregation.

@Nullable
@Override
public Aggregation toDruidAggregation(final PlannerContext plannerContext, final RowSignature rowSignature, final VirtualColumnRegistry virtualColumnRegistry, final RexBuilder rexBuilder, final String name, final AggregateCall aggregateCall, final Project project, final List<Aggregation> existingAggregations, final boolean finalizeAggregations) {
    // This is expected to be a tdigest sketch
    final DruidExpression input = Aggregations.toDruidExpressionForNumericAggregator(plannerContext, rowSignature, Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(0)));
    if (input == null) {
        return null;
    }
    final AggregatorFactory aggregatorFactory;
    final String sketchName = StringUtils.format("%s:agg", name);
    // this is expected to be quantile fraction
    final RexNode quantileArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(1));
    if (!quantileArg.isA(SqlKind.LITERAL)) {
        // Quantile must be a literal in order to plan.
        return null;
    }
    final double quantile = ((Number) RexLiteral.value(quantileArg)).floatValue();
    Integer compression = TDigestSketchAggregatorFactory.DEFAULT_COMPRESSION;
    if (aggregateCall.getArgList().size() > 2) {
        final RexNode compressionArg = Expressions.fromFieldAccess(rowSignature, project, aggregateCall.getArgList().get(2));
        compression = ((Number) RexLiteral.value(compressionArg)).intValue();
    }
    // Look for existing matching aggregatorFactory.
    for (final Aggregation existing : existingAggregations) {
        for (AggregatorFactory factory : existing.getAggregatorFactories()) {
            if (factory instanceof TDigestSketchAggregatorFactory) {
                final boolean matches = TDigestSketchUtils.matchingAggregatorFactoryExists(virtualColumnRegistry, input, compression, (TDigestSketchAggregatorFactory) factory);
                if (matches) {
                    // Found existing one. Use this.
                    return Aggregation.create(ImmutableList.of(), new TDigestSketchToQuantilePostAggregator(name, new FieldAccessPostAggregator(factory.getName(), factory.getName()), quantile));
                }
            }
        }
    }
    // No existing match found. Create a new one.
    if (input.isDirectColumnAccess()) {
        aggregatorFactory = new TDigestSketchAggregatorFactory(sketchName, input.getDirectColumn(), compression);
    } else {
        String virtualColumnName = virtualColumnRegistry.getOrCreateVirtualColumnForExpression(input, ColumnType.FLOAT);
        aggregatorFactory = new TDigestSketchAggregatorFactory(sketchName, virtualColumnName, compression);
    }
    return Aggregation.create(ImmutableList.of(aggregatorFactory), new TDigestSketchToQuantilePostAggregator(name, new FieldAccessPostAggregator(sketchName, sketchName), quantile));
}

Example 2 with TDigestSketchToQuantilePostAggregator

use of org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchToQuantilePostAggregator in project druid by druid-io.

the class TDigestSketchSqlAggregatorTest method testGeneratingSketchAndComputingQuantileOnFly.

@Test
public void testGeneratingSketchAndComputingQuantileOnFly() throws Exception {
    cannotVectorize();
    final List<Object[]> expectedResults = ImmutableList.of(new Object[] { 1.0, 3.5, 6.0 });
    testQuery("SELECT TDIGEST_QUANTILE(x, 0.0), TDIGEST_QUANTILE(x, 0.5), TDIGEST_QUANTILE(x, 1.0)\n" + "FROM (SELECT dim1, TDIGEST_GENERATE_SKETCH(m1, 200) AS x FROM foo group by dim1)", ImmutableList.of(GroupByQuery.builder().setDataSource(new QueryDataSource(GroupByQuery.builder().setDataSource(CalciteTests.DATASOURCE1).setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("dim1", "d0")).setAggregatorSpecs(ImmutableList.of(new TDigestSketchAggregatorFactory("a0:agg", "m1", 200))).setContext(QUERY_CONTEXT_DEFAULT).build())).setInterval(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).setGranularity(Granularities.ALL).setAggregatorSpecs(ImmutableList.of(new TDigestSketchAggregatorFactory("_a0:agg", "a0:agg", 100))).setPostAggregatorSpecs(ImmutableList.of(new TDigestSketchToQuantilePostAggregator("_a0", makeFieldAccessPostAgg("_a0:agg"), 0.0f), new TDigestSketchToQuantilePostAggregator("_a1", makeFieldAccessPostAgg("_a0:agg"), 0.5f), new TDigestSketchToQuantilePostAggregator("_a2", makeFieldAccessPostAgg("_a0:agg"), 1.0f))).setContext(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}

Example 3 with TDigestSketchToQuantilePostAggregator

use of org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchToQuantilePostAggregator in project druid by druid-io.

the class TDigestSketchSqlAggregatorTest method testComputingQuantileOnPreAggregatedSketch.

@Test
public void testComputingQuantileOnPreAggregatedSketch() throws Exception {
    cannotVectorize();
    final List<Object[]> expectedResults = ImmutableList.of(new Object[] { 1.1, 2.9, 5.3, 6.0 });
    testQuery("SELECT\n" + "TDIGEST_QUANTILE(qsketch_m1, 0.1),\n" + "TDIGEST_QUANTILE(qsketch_m1, 0.4),\n" + "TDIGEST_QUANTILE(qsketch_m1, 0.8),\n" + "TDIGEST_QUANTILE(qsketch_m1, 1.0)\n" + "FROM foo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).aggregators(ImmutableList.of(new TDigestSketchAggregatorFactory("a0:agg", "qsketch_m1", 100))).postAggregators(new TDigestSketchToQuantilePostAggregator("a0", makeFieldAccessPostAgg("a0:agg"), 0.1f), new TDigestSketchToQuantilePostAggregator("a1", makeFieldAccessPostAgg("a0:agg"), 0.4f), new TDigestSketchToQuantilePostAggregator("a2", makeFieldAccessPostAgg("a0:agg"), 0.8f), new TDigestSketchToQuantilePostAggregator("a3", makeFieldAccessPostAgg("a0:agg"), 1.0f)).context(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}

Example 4 with TDigestSketchToQuantilePostAggregator

use of org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchToQuantilePostAggregator in project druid by druid-io.

the class TDigestSketchSqlAggregatorTest method testQuantileOnNumericValues.

@Test
public void testQuantileOnNumericValues() throws Exception {
    cannotVectorize();
    final List<Object[]> expectedResults = ImmutableList.of(new Object[] { 1.0, 3.5, 6.0 });
    testQuery("SELECT\n" + "TDIGEST_QUANTILE(m1, 0.0), TDIGEST_QUANTILE(m1, 0.5), TDIGEST_QUANTILE(m1, 1.0)\n" + "FROM foo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).aggregators(ImmutableList.of(new TDigestSketchAggregatorFactory("a0:agg", "m1", null))).postAggregators(new TDigestSketchToQuantilePostAggregator("a0", makeFieldAccessPostAgg("a0:agg"), 0.0f), new TDigestSketchToQuantilePostAggregator("a1", makeFieldAccessPostAgg("a0:agg"), 0.5f), new TDigestSketchToQuantilePostAggregator("a2", makeFieldAccessPostAgg("a0:agg"), 1.0f)).context(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}

Example 5 with TDigestSketchToQuantilePostAggregator

use of org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchToQuantilePostAggregator in project druid by druid-io.

the class TDigestSketchSqlAggregatorTest method testQuantileOnCastedString.

@Test
public void testQuantileOnCastedString() throws Exception {
    cannotVectorize();
    testQuery("SELECT\n" + "  TDIGEST_QUANTILE(CAST(dim1 AS DOUBLE), 0.0),\n" + "  TDIGEST_QUANTILE(CAST(dim1 AS DOUBLE), 0.5),\n" + "  TDIGEST_QUANTILE(CAST(dim1 AS DOUBLE), 1.0)\n" + "FROM foo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(querySegmentSpec(Filtration.eternity())).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "CAST(\"dim1\", 'DOUBLE')", ColumnType.FLOAT, ExprMacroTable.nil())).aggregators(new TDigestSketchAggregatorFactory("a0:agg", "v0", 100)).postAggregators(new TDigestSketchToQuantilePostAggregator("a0", new FieldAccessPostAggregator("a0:agg", "a0:agg"), 0.0), new TDigestSketchToQuantilePostAggregator("a1", new FieldAccessPostAggregator("a0:agg", "a0:agg"), 0.5), new TDigestSketchToQuantilePostAggregator("a2", new FieldAccessPostAggregator("a0:agg", "a0:agg"), 1.0)).context(QUERY_CONTEXT_DEFAULT).build()), ImmutableList.of(NullHandling.replaceWithDefault() ? new Object[] { 0.0, 0.5, 10.1 } : new Object[] { 1.0, 2.0, 10.1 }));
}