Search in sources :

Example 41 with OrderByColumnSpec

use of org.apache.druid.query.groupby.orderby.OrderByColumnSpec in project druid by druid-io.

the class GroupByQueryRunnerTest method testSubqueryWithMultiColumnAggregators.

@Test
public void testSubqueryWithMultiColumnAggregators() {
    // Cannot vectorize due to javascript functionality.
    cannotVectorize();
    final GroupByQuery subquery = makeQueryBuilder().setDataSource(QueryRunnerTestHelper.DATA_SOURCE).setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD).setDimensions(new DefaultDimensionSpec("quality", "alias")).setDimFilter(new JavaScriptDimFilter("market", "function(dim){ return true; }", null, JavaScriptConfig.getEnabledInstance())).setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, new DoubleSumAggregatorFactory("idx_subagg", "index"), new JavaScriptAggregatorFactory("js_agg", Arrays.asList("index", "market"), "function(current, index, dim){return current + index + dim.length;}", "function(){return 0;}", "function(a,b){return a + b;}", JavaScriptConfig.getEnabledInstance())).setPostAggregatorSpecs(Collections.singletonList(new ArithmeticPostAggregator("idx_subpostagg", "+", Arrays.asList(new FieldAccessPostAggregator("the_idx_subagg", "idx_subagg"), new ConstantPostAggregator("thousand", 1000))))).setHavingSpec(new HavingSpec() {

        private GroupByQuery query;

        @Override
        public byte[] getCacheKey() {
            return new byte[0];
        }

        @Override
        public void setQuery(GroupByQuery query) {
            this.query = query;
        }

        @Override
        public boolean eval(ResultRow row) {
            final String field = "idx_subpostagg";
            final int p = query.getResultRowSignature().indexOf(field);
            return (Rows.objectToNumber(field, row.get(p), true).floatValue() < 3800);
        }
    }).addOrderByColumn("alias").setGranularity(QueryRunnerTestHelper.DAY_GRAN).build();
    final GroupByQuery query = makeQueryBuilder().setDataSource(subquery).setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD).setDimensions(new DefaultDimensionSpec("alias", "alias")).setAggregatorSpecs(new LongSumAggregatorFactory("rows", "rows"), new LongSumAggregatorFactory("idx", "idx_subpostagg"), new DoubleSumAggregatorFactory("js_outer_agg", "js_agg")).setPostAggregatorSpecs(Collections.singletonList(new ArithmeticPostAggregator("idx_post", "+", Arrays.asList(new FieldAccessPostAggregator("the_idx_agg", "idx"), new ConstantPostAggregator("ten_thousand", 10000))))).setLimitSpec(new DefaultLimitSpec(Collections.singletonList(new OrderByColumnSpec("alias", OrderByColumnSpec.Direction.DESCENDING)), 5)).setGranularity(QueryRunnerTestHelper.DAY_GRAN).build();
    List<ResultRow> expectedResults = Arrays.asList(makeRow(query, "2011-04-01", "alias", "travel", "rows", 1L, "idx_post", 11119.0, "idx", 1119L, "js_outer_agg", 123.92274475097656), makeRow(query, "2011-04-01", "alias", "technology", "rows", 1L, "idx_post", 11078.0, "idx", 1078L, "js_outer_agg", 82.62254333496094), makeRow(query, "2011-04-01", "alias", "news", "rows", 1L, "idx_post", 11121.0, "idx", 1121L, "js_outer_agg", 125.58358001708984), makeRow(query, "2011-04-01", "alias", "health", "rows", 1L, "idx_post", 11120.0, "idx", 1120L, "js_outer_agg", 124.13470458984375), makeRow(query, "2011-04-01", "alias", "entertainment", "rows", 1L, "idx_post", 11158.0, "idx", 1158L, "js_outer_agg", 162.74722290039062));
    // Subqueries are handled by the ToolChest
    Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
    TestHelper.assertExpectedObjects(expectedResults, results, "subquery-multi-aggs");
}
Also used : ArithmeticPostAggregator(org.apache.druid.query.aggregation.post.ArithmeticPostAggregator) FieldAccessPostAggregator(org.apache.druid.query.aggregation.post.FieldAccessPostAggregator) DoubleSumAggregatorFactory(org.apache.druid.query.aggregation.DoubleSumAggregatorFactory) DefaultLimitSpec(org.apache.druid.query.groupby.orderby.DefaultLimitSpec) ConstantPostAggregator(org.apache.druid.query.aggregation.post.ConstantPostAggregator) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) JavaScriptAggregatorFactory(org.apache.druid.query.aggregation.JavaScriptAggregatorFactory) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) OrderByColumnSpec(org.apache.druid.query.groupby.orderby.OrderByColumnSpec) DimFilterHavingSpec(org.apache.druid.query.groupby.having.DimFilterHavingSpec) EqualToHavingSpec(org.apache.druid.query.groupby.having.EqualToHavingSpec) DimensionSelectorHavingSpec(org.apache.druid.query.groupby.having.DimensionSelectorHavingSpec) HavingSpec(org.apache.druid.query.groupby.having.HavingSpec) OrHavingSpec(org.apache.druid.query.groupby.having.OrHavingSpec) GreaterThanHavingSpec(org.apache.druid.query.groupby.having.GreaterThanHavingSpec) JavaScriptDimFilter(org.apache.druid.query.filter.JavaScriptDimFilter) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 42 with OrderByColumnSpec

use of org.apache.druid.query.groupby.orderby.OrderByColumnSpec in project druid by druid-io.

the class GroupByQueryRunnerTest method testGroupByWithLimitOnFinalizedHyperUnique.

@Test
public void testGroupByWithLimitOnFinalizedHyperUnique() {
    GroupByQuery query = makeQueryBuilder().setDataSource(QueryRunnerTestHelper.DATA_SOURCE).setGranularity(QueryRunnerTestHelper.ALL_GRAN).setDimensions(new DefaultDimensionSpec(QueryRunnerTestHelper.MARKET_DIMENSION, QueryRunnerTestHelper.MARKET_DIMENSION)).setInterval(QueryRunnerTestHelper.FULL_ON_INTERVAL_SPEC).setLimitSpec(new DefaultLimitSpec(Collections.singletonList(new OrderByColumnSpec(QueryRunnerTestHelper.HYPER_UNIQUE_FINALIZING_POST_AGG_METRIC, OrderByColumnSpec.Direction.DESCENDING)), 3)).setAggregatorSpecs(QueryRunnerTestHelper.QUALITY_UNIQUES).setPostAggregatorSpecs(Collections.singletonList(new HyperUniqueFinalizingPostAggregator(QueryRunnerTestHelper.HYPER_UNIQUE_FINALIZING_POST_AGG_METRIC, QueryRunnerTestHelper.UNIQUE_METRIC))).build();
    List<ResultRow> expectedResults = Arrays.asList(makeRow(query, "1970-01-01T00:00:00.000Z", "market", "spot", QueryRunnerTestHelper.UNIQUE_METRIC, QueryRunnerTestHelper.UNIQUES_9, QueryRunnerTestHelper.HYPER_UNIQUE_FINALIZING_POST_AGG_METRIC, QueryRunnerTestHelper.UNIQUES_9), makeRow(query, "1970-01-01T00:00:00.000Z", "market", "total_market", QueryRunnerTestHelper.UNIQUE_METRIC, QueryRunnerTestHelper.UNIQUES_2, QueryRunnerTestHelper.HYPER_UNIQUE_FINALIZING_POST_AGG_METRIC, QueryRunnerTestHelper.UNIQUES_2), makeRow(query, "1970-01-01T00:00:00.000Z", "market", "upfront", QueryRunnerTestHelper.UNIQUE_METRIC, QueryRunnerTestHelper.UNIQUES_2, QueryRunnerTestHelper.HYPER_UNIQUE_FINALIZING_POST_AGG_METRIC, QueryRunnerTestHelper.UNIQUES_2));
    Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
    TestHelper.assertExpectedObjects(expectedResults, results, "order-limit");
}
Also used : OrderByColumnSpec(org.apache.druid.query.groupby.orderby.OrderByColumnSpec) DefaultLimitSpec(org.apache.druid.query.groupby.orderby.DefaultLimitSpec) HyperUniqueFinalizingPostAggregator(org.apache.druid.query.aggregation.hyperloglog.HyperUniqueFinalizingPostAggregator) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 43 with OrderByColumnSpec

use of org.apache.druid.query.groupby.orderby.OrderByColumnSpec in project druid by druid-io.

the class GroupByQueryRunnerTest method testMergeResultsWithLimitPushDownSortByAgg.

@Test
public void testMergeResultsWithLimitPushDownSortByAgg() {
    if (!config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V2)) {
        return;
    }
    GroupByQuery.Builder builder = makeQueryBuilder().setDataSource(QueryRunnerTestHelper.DATA_SOURCE).setInterval("2011-04-02/2011-04-04").setDimensions(new DefaultDimensionSpec("quality", "alias")).setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, new LongSumAggregatorFactory("idx", "index")).setLimitSpec(new DefaultLimitSpec(Collections.singletonList(new OrderByColumnSpec("idx", OrderByColumnSpec.Direction.DESCENDING)), 5)).overrideContext(ImmutableMap.of(GroupByQueryConfig.CTX_KEY_FORCE_LIMIT_PUSH_DOWN, true)).setGranularity(Granularities.ALL);
    final GroupByQuery allGranQuery = builder.build();
    QueryRunner mergedRunner = factory.getToolchest().mergeResults(new QueryRunner<ResultRow>() {

        @Override
        public Sequence<ResultRow> run(QueryPlus<ResultRow> queryPlus, ResponseContext responseContext) {
            // simulate two daily segments
            final QueryPlus<ResultRow> queryPlus1 = queryPlus.withQuery(queryPlus.getQuery().withQuerySegmentSpec(new MultipleIntervalSegmentSpec(Collections.singletonList(Intervals.of("2011-04-02/2011-04-03")))));
            final QueryPlus<ResultRow> queryPlus2 = queryPlus.withQuery(queryPlus.getQuery().withQuerySegmentSpec(new MultipleIntervalSegmentSpec(Collections.singletonList(Intervals.of("2011-04-03/2011-04-04")))));
            return factory.getToolchest().mergeResults((queryPlus3, responseContext1) -> new MergeSequence<>(queryPlus3.getQuery().getResultOrdering(), Sequences.simple(Arrays.asList(runner.run(queryPlus1, responseContext1), runner.run(queryPlus2, responseContext1))))).run(queryPlus, responseContext);
        }
    });
    List<ResultRow> allGranExpectedResults = Arrays.asList(makeRow(allGranQuery, "2011-04-02", "alias", "mezzanine", "rows", 6L, "idx", 4420L), makeRow(allGranQuery, "2011-04-02", "alias", "premium", "rows", 6L, "idx", 4416L), makeRow(allGranQuery, "2011-04-02", "alias", "entertainment", "rows", 2L, "idx", 319L), makeRow(allGranQuery, "2011-04-02", "alias", "automotive", "rows", 2L, "idx", 269L), makeRow(allGranQuery, "2011-04-02", "alias", "travel", "rows", 2L, "idx", 243L));
    Iterable<ResultRow> results = mergedRunner.run(QueryPlus.wrap(allGranQuery)).toList();
    TestHelper.assertExpectedObjects(allGranExpectedResults, results, "merged");
}
Also used : DefaultLimitSpec(org.apache.druid.query.groupby.orderby.DefaultLimitSpec) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) MultipleIntervalSegmentSpec(org.apache.druid.query.spec.MultipleIntervalSegmentSpec) Sequence(org.apache.druid.java.util.common.guava.Sequence) MergeSequence(org.apache.druid.java.util.common.guava.MergeSequence) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) FinalizeResultsQueryRunner(org.apache.druid.query.FinalizeResultsQueryRunner) ChainedExecutionQueryRunner(org.apache.druid.query.ChainedExecutionQueryRunner) QueryRunner(org.apache.druid.query.QueryRunner) OrderByColumnSpec(org.apache.druid.query.groupby.orderby.OrderByColumnSpec) ResponseContext(org.apache.druid.query.context.ResponseContext) QueryPlus(org.apache.druid.query.QueryPlus) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 44 with OrderByColumnSpec

use of org.apache.druid.query.groupby.orderby.OrderByColumnSpec in project druid by druid-io.

the class GroupByQueryRunnerTest method testGroupByDoubleColumnDescending.

@Test
public void testGroupByDoubleColumnDescending() {
    if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
        expectedException.expect(UnsupportedOperationException.class);
        expectedException.expectMessage("GroupBy v1 only supports dimensions with an outputType of STRING.");
    }
    GroupByQuery query = makeQueryBuilder().setDataSource(QueryRunnerTestHelper.DATA_SOURCE).setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD).setDimensions(new DefaultDimensionSpec("qualityDouble", "alias", ColumnType.DOUBLE)).setDimFilter(new InDimFilter("quality", Arrays.asList("entertainment", "technology"), null)).setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT, new LongSumAggregatorFactory("idx", "index")).addOrderByColumn(new OrderByColumnSpec("alias", OrderByColumnSpec.Direction.DESCENDING, StringComparators.NUMERIC)).setGranularity(QueryRunnerTestHelper.ALL_GRAN).build();
    Assert.assertNotEquals(Functions.<Sequence<ResultRow>>identity(), query.getLimitSpec().build(query));
    List<ResultRow> expectedResults = Arrays.asList(makeRow(query, "2011-04-01", "alias", 17000.0d, "rows", 2L, "idx", 175L), makeRow(query, "2011-04-01", "alias", 12000.0d, "rows", 2L, "idx", 324L));
    Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, query);
    TestHelper.assertExpectedObjects(expectedResults, results, "double");
}
Also used : OrderByColumnSpec(org.apache.druid.query.groupby.orderby.OrderByColumnSpec) InDimFilter(org.apache.druid.query.filter.InDimFilter) LongSumAggregatorFactory(org.apache.druid.query.aggregation.LongSumAggregatorFactory) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 45 with OrderByColumnSpec

use of org.apache.druid.query.groupby.orderby.OrderByColumnSpec in project druid by druid-io.

the class GroupByQueryRunnerTest method testGroupByWithLongArraysDesc.

@Test
public void testGroupByWithLongArraysDesc() {
    if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
        expectedException.expect(UnsupportedOperationException.class);
        expectedException.expectMessage("GroupBy v1 only supports dimensions with an outputType of STRING");
    }
    cannotVectorize();
    GroupByQuery outer = makeQueryBuilder().setDataSource(QueryRunnerTestHelper.DATA_SOURCE).setQuerySegmentSpec(QueryRunnerTestHelper.FIRST_TO_THIRD).setVirtualColumns(new ExpressionVirtualColumn("v0", "array(index)", ColumnType.LONG_ARRAY, ExprMacroTable.nil())).setDimensions(new DefaultDimensionSpec("v0", "alias_outer", ColumnType.LONG_ARRAY)).setLimitSpec(new DefaultLimitSpec(ImmutableList.of(new OrderByColumnSpec("alias_outer", OrderByColumnSpec.Direction.DESCENDING, StringComparators.NUMERIC)), Integer.MAX_VALUE - 1)).setAggregatorSpecs(QueryRunnerTestHelper.ROWS_COUNT).setGranularity(QueryRunnerTestHelper.ALL_GRAN).build();
    List<ResultRow> expectedResults = Arrays.asList(makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(78.622547)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(97.387433)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(109.705815)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(110.931934)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(112.987027)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(113.446008)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(114.290141)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(118.57034)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(119.922742)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(120.134704)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(121.583581)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(126.411364)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(135.301506)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(135.885094)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(144.507368)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(147.425935)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(158.747224)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(166.016049)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(1049.738585)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(1144.342401)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(1193.556278)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(1234.247546)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(1314.839715)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(1321.375057)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(1447.34116)), "rows", 1L), makeRow(outer, "2011-04-01", "alias_outer", new ComparableList(ImmutableList.of(1522.043733)), "rows", 1L));
    // reversing list
    Collections.reverse(expectedResults);
    Iterable<ResultRow> results = GroupByQueryRunnerTestHelper.runQuery(factory, runner, outer);
    TestHelper.assertExpectedObjects(expectedResults, results, "long-groupby-arrays");
}
Also used : OrderByColumnSpec(org.apache.druid.query.groupby.orderby.OrderByColumnSpec) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) DefaultLimitSpec(org.apache.druid.query.groupby.orderby.DefaultLimitSpec) ComparableList(org.apache.druid.segment.data.ComparableList) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Aggregations

OrderByColumnSpec (org.apache.druid.query.groupby.orderby.OrderByColumnSpec)111 DefaultDimensionSpec (org.apache.druid.query.dimension.DefaultDimensionSpec)105 Test (org.junit.Test)104 DefaultLimitSpec (org.apache.druid.query.groupby.orderby.DefaultLimitSpec)99 LongSumAggregatorFactory (org.apache.druid.query.aggregation.LongSumAggregatorFactory)79 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)46 MultipleIntervalSegmentSpec (org.apache.druid.query.spec.MultipleIntervalSegmentSpec)15 FinalizeResultsQueryRunner (org.apache.druid.query.FinalizeResultsQueryRunner)11 QueryPlus (org.apache.druid.query.QueryPlus)10 QueryRunner (org.apache.druid.query.QueryRunner)10 ResponseContext (org.apache.druid.query.context.ResponseContext)10 CountAggregatorFactory (org.apache.druid.query.aggregation.CountAggregatorFactory)9 GreaterThanHavingSpec (org.apache.druid.query.groupby.having.GreaterThanHavingSpec)8 QuerySegmentSpec (org.apache.druid.query.spec.QuerySegmentSpec)8 ExpressionVirtualColumn (org.apache.druid.segment.virtual.ExpressionVirtualColumn)7 ListFilteredVirtualColumn (org.apache.druid.segment.virtual.ListFilteredVirtualColumn)7 ArrayList (java.util.ArrayList)6 Sequence (org.apache.druid.java.util.common.guava.Sequence)6 QueryDataSource (org.apache.druid.query.QueryDataSource)6 GroupByQuery (org.apache.druid.query.groupby.GroupByQuery)6