Search in sources :

Example 61 with FilteredAggregatorFactory

use of org.apache.druid.query.aggregation.FilteredAggregatorFactory in project druid by druid-io.

the class CalciteJoinQueryTest method testJoinOuterGroupByAndSubqueryHasLimit.

@Test
public void testJoinOuterGroupByAndSubqueryHasLimit() throws Exception {
    // Cannot vectorize JOIN operator.
    cannotVectorize();
    testQuery("SELECT dim2, AVG(m2) FROM (SELECT * FROM foo AS t1 INNER JOIN foo AS t2 ON t1.m1 = t2.m1 LIMIT 10) AS t3 GROUP BY dim2", ImmutableList.of(GroupByQuery.builder().setDataSource(newScanQueryBuilder().dataSource(join(new TableDataSource(CalciteTests.DATASOURCE1), new QueryDataSource(newScanQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(querySegmentSpec(Filtration.eternity())).columns(ImmutableList.of("m1")).resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST).context(QUERY_CONTEXT_DEFAULT).build()), "j0.", equalsCondition(DruidExpression.ofColumn(ColumnType.FLOAT, "m1"), DruidExpression.ofColumn(ColumnType.FLOAT, "j0.m1")), JoinType.INNER)).intervals(querySegmentSpec(Filtration.eternity())).limit(10).columns("dim2", "m2").context(QUERY_CONTEXT_DEFAULT).build()).setInterval(querySegmentSpec(Filtration.eternity())).setDimensions(new DefaultDimensionSpec("dim2", "d0", ColumnType.STRING)).setGranularity(Granularities.ALL).setAggregatorSpecs(useDefault ? aggregators(new DoubleSumAggregatorFactory("a0:sum", "m2"), new CountAggregatorFactory("a0:count")) : aggregators(new DoubleSumAggregatorFactory("a0:sum", "m2"), new FilteredAggregatorFactory(new CountAggregatorFactory("a0:count"), not(selector("m2", null, null))))).setPostAggregatorSpecs(ImmutableList.of(new ArithmeticPostAggregator("a0", "quotient", ImmutableList.of(new FieldAccessPostAggregator(null, "a0:sum"), new FieldAccessPostAggregator(null, "a0:count"))))).setContext(QUERY_CONTEXT_DEFAULT).build()), NullHandling.sqlCompatible() ? ImmutableList.of(new Object[] { null, 4.0 }, new Object[] { "", 3.0 }, new Object[] { "a", 2.5 }, new Object[] { "abc", 5.0 }) : ImmutableList.of(new Object[] { "", 3.6666666666666665 }, new Object[] { "a", 2.5 }, new Object[] { "abc", 5.0 }));
}
Also used : FilteredAggregatorFactory(org.apache.druid.query.aggregation.FilteredAggregatorFactory) ArithmeticPostAggregator(org.apache.druid.query.aggregation.post.ArithmeticPostAggregator) FieldAccessPostAggregator(org.apache.druid.query.aggregation.post.FieldAccessPostAggregator) GlobalTableDataSource(org.apache.druid.query.GlobalTableDataSource) TableDataSource(org.apache.druid.query.TableDataSource) QueryDataSource(org.apache.druid.query.QueryDataSource) DoubleSumAggregatorFactory(org.apache.druid.query.aggregation.DoubleSumAggregatorFactory) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) Test(org.junit.Test)

Example 62 with FilteredAggregatorFactory

use of org.apache.druid.query.aggregation.FilteredAggregatorFactory in project druid by druid-io.

the class CalciteCorrelatedQueryTest method testCorrelatedSubqueryWithCorrelatedQueryFilter_Scan.

@Test
@Parameters(source = QueryContextForJoinProvider.class)
public void testCorrelatedSubqueryWithCorrelatedQueryFilter_Scan(Map<String, Object> queryContext) throws Exception {
    cannotVectorize();
    queryContext = withLeftDirectAccessEnabled(queryContext);
    testQuery("select country, ANY_VALUE(\n" + "        select max(\"users\") from (\n" + "            select floor(__time to day), count(user) \"users\" from visits f where f.country = visits.country and f.city = 'A' group by 1\n" + "        )\n" + "     ) as \"dailyVisits\"\n" + "from visits \n" + " where city = 'B'" + " group by 1", queryContext, ImmutableList.of(GroupByQuery.builder().setDataSource(join(new TableDataSource(CalciteTests.USERVISITDATASOURCE), new QueryDataSource(GroupByQuery.builder().setDataSource(GroupByQuery.builder().setDataSource(CalciteTests.USERVISITDATASOURCE).setQuerySegmentSpec(querySegmentSpec(Intervals.ETERNITY)).setVirtualColumns(new ExpressionVirtualColumn("v0", "timestamp_floor(\"__time\",'P1D',null,'UTC')", ColumnType.LONG, TestExprMacroTable.INSTANCE)).setDimensions(new DefaultDimensionSpec("v0", "d0", ColumnType.LONG), new DefaultDimensionSpec("country", "d1")).setAggregatorSpecs(new FilteredAggregatorFactory(new CountAggregatorFactory("a0"), not(selector("user", null, null)))).setDimFilter(and(selector("city", "A", null), not(selector("country", null, null)))).setContext(withTimestampResultContext(queryContext, "d0", Granularities.DAY)).setGranularity(new AllGranularity()).build()).setQuerySegmentSpec(querySegmentSpec(Intervals.ETERNITY)).setDimensions(new DefaultDimensionSpec("d1", "_d0")).setAggregatorSpecs(new LongMaxAggregatorFactory("_a0", "a0")).setGranularity(new AllGranularity()).setContext(queryContext).build()), "j0.", equalsCondition(makeColumnExpression("country"), makeColumnExpression("j0._d0")), JoinType.LEFT, selector("city", "B", null))).setQuerySegmentSpec(querySegmentSpec(Intervals.ETERNITY)).setDimensions(new DefaultDimensionSpec("country", "d0")).setAggregatorSpecs(new LongAnyAggregatorFactory("a0", "j0._a0")).setGranularity(new AllGranularity()).setContext(queryContext).build()), ImmutableList.of(new Object[] { "canada", 2L }));
}
Also used : FilteredAggregatorFactory(org.apache.druid.query.aggregation.FilteredAggregatorFactory) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) TableDataSource(org.apache.druid.query.TableDataSource) QueryDataSource(org.apache.druid.query.QueryDataSource) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) LongAnyAggregatorFactory(org.apache.druid.query.aggregation.any.LongAnyAggregatorFactory) AllGranularity(org.apache.druid.java.util.common.granularity.AllGranularity) LongMaxAggregatorFactory(org.apache.druid.query.aggregation.LongMaxAggregatorFactory) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) Parameters(junitparams.Parameters) Test(org.junit.Test)

Example 63 with FilteredAggregatorFactory

use of org.apache.druid.query.aggregation.FilteredAggregatorFactory in project druid by druid-io.

the class GroupByRules method translateAggregateCall.

/**
 * Translate an AggregateCall to Druid equivalents.
 *
 * @return translated aggregation, or null if translation failed.
 */
public static Aggregation translateAggregateCall(final PlannerContext plannerContext, final RowSignature rowSignature, final VirtualColumnRegistry virtualColumnRegistry, final RexBuilder rexBuilder, final Project project, final List<Aggregation> existingAggregations, final String name, final AggregateCall call, final boolean finalizeAggregations) {
    final DimFilter filter;
    if (call.filterArg >= 0) {
        // AGG(xxx) FILTER(WHERE yyy)
        if (project == null) {
            // We need some kind of projection to support filtered aggregations.
            return null;
        }
        final RexNode expression = project.getChildExps().get(call.filterArg);
        final DimFilter nonOptimizedFilter = Expressions.toFilter(plannerContext, rowSignature, virtualColumnRegistry, expression);
        if (nonOptimizedFilter == null) {
            return null;
        } else {
            filter = Filtration.create(nonOptimizedFilter).optimizeFilterOnly(virtualColumnRegistry.getFullRowSignature()).getDimFilter();
        }
    } else {
        filter = null;
    }
    final SqlAggregator sqlAggregator = plannerContext.getOperatorTable().lookupAggregator(call.getAggregation());
    if (sqlAggregator == null) {
        return null;
    }
    // Compute existingAggregations for SqlAggregator impls that want it.
    final List<Aggregation> existingAggregationsWithSameFilter = new ArrayList<>();
    for (Aggregation existingAggregation : existingAggregations) {
        if (filter == null) {
            final boolean doesMatch = existingAggregation.getAggregatorFactories().stream().noneMatch(factory -> factory instanceof FilteredAggregatorFactory);
            if (doesMatch) {
                existingAggregationsWithSameFilter.add(existingAggregation);
            }
        } else {
            final boolean doesMatch = existingAggregation.getAggregatorFactories().stream().allMatch(factory -> factory instanceof FilteredAggregatorFactory && ((FilteredAggregatorFactory) factory).getFilter().equals(filter));
            if (doesMatch) {
                existingAggregationsWithSameFilter.add(Aggregation.create(existingAggregation.getAggregatorFactories().stream().map(factory -> ((FilteredAggregatorFactory) factory).getAggregator()).collect(Collectors.toList()), existingAggregation.getPostAggregator()));
            }
        }
    }
    final Aggregation retVal = sqlAggregator.toDruidAggregation(plannerContext, rowSignature, virtualColumnRegistry, rexBuilder, name, call, project, existingAggregationsWithSameFilter, finalizeAggregations);
    if (retVal == null) {
        return null;
    } else {
        // Check if this refers to the existingAggregationsWithSameFilter. If so, no need to apply the filter.
        if (isUsingExistingAggregation(retVal, existingAggregationsWithSameFilter)) {
            return retVal;
        } else {
            return retVal.filter(rowSignature, virtualColumnRegistry, filter);
        }
    }
}
Also used : Aggregation(org.apache.druid.sql.calcite.aggregation.Aggregation) FilteredAggregatorFactory(org.apache.druid.query.aggregation.FilteredAggregatorFactory) Project(org.apache.calcite.rel.core.Project) RexBuilder(org.apache.calcite.rex.RexBuilder) SqlAggregator(org.apache.druid.sql.calcite.aggregation.SqlAggregator) AggregatorFactory(org.apache.druid.query.aggregation.AggregatorFactory) FilteredAggregatorFactory(org.apache.druid.query.aggregation.FilteredAggregatorFactory) Set(java.util.Set) Aggregation(org.apache.druid.sql.calcite.aggregation.Aggregation) Collectors(java.util.stream.Collectors) ArrayList(java.util.ArrayList) List(java.util.List) DimFilter(org.apache.druid.query.filter.DimFilter) RexNode(org.apache.calcite.rex.RexNode) RowSignature(org.apache.druid.segment.column.RowSignature) VirtualColumnRegistry(org.apache.druid.sql.calcite.rel.VirtualColumnRegistry) PlannerContext(org.apache.druid.sql.calcite.planner.PlannerContext) AggregateCall(org.apache.calcite.rel.core.AggregateCall) Expressions(org.apache.druid.sql.calcite.expression.Expressions) Filtration(org.apache.druid.sql.calcite.filtration.Filtration) SqlAggregator(org.apache.druid.sql.calcite.aggregation.SqlAggregator) ArrayList(java.util.ArrayList) DimFilter(org.apache.druid.query.filter.DimFilter) RexNode(org.apache.calcite.rex.RexNode)

Aggregations

FilteredAggregatorFactory (org.apache.druid.query.aggregation.FilteredAggregatorFactory)63 Test (org.junit.Test)52 CountAggregatorFactory (org.apache.druid.query.aggregation.CountAggregatorFactory)28 LongSumAggregatorFactory (org.apache.druid.query.aggregation.LongSumAggregatorFactory)23 DefaultDimensionSpec (org.apache.druid.query.dimension.DefaultDimensionSpec)23 SelectorDimFilter (org.apache.druid.query.filter.SelectorDimFilter)20 QueryDataSource (org.apache.druid.query.QueryDataSource)14 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)14 ExpressionLambdaAggregatorFactory (org.apache.druid.query.aggregation.ExpressionLambdaAggregatorFactory)13 NotDimFilter (org.apache.druid.query.filter.NotDimFilter)11 AggregatorFactory (org.apache.druid.query.aggregation.AggregatorFactory)10 BaseCalciteQueryTest (org.apache.druid.sql.calcite.BaseCalciteQueryTest)10 ArithmeticPostAggregator (org.apache.druid.query.aggregation.post.ArithmeticPostAggregator)8 FieldAccessPostAggregator (org.apache.druid.query.aggregation.post.FieldAccessPostAggregator)8 BoundDimFilter (org.apache.druid.query.filter.BoundDimFilter)8 MultipleIntervalSegmentSpec (org.apache.druid.query.spec.MultipleIntervalSegmentSpec)8 Result (org.apache.druid.query.Result)7 LongMaxAggregatorFactory (org.apache.druid.query.aggregation.LongMaxAggregatorFactory)7 TableDataSource (org.apache.druid.query.TableDataSource)6 DimFilter (org.apache.druid.query.filter.DimFilter)6