Search in sources :

Example 31 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class MultiValuedDimensionTest method testGroupByExpressionAuto.

@Test
public void testGroupByExpressionAuto() {
    // virtual column is a single input column and input is not used explicitly as an array,
    // so this one will work for group by v1, even with multi-value inputs
    GroupByQuery query = GroupByQuery.builder().setDataSource("xx").setQuerySegmentSpec(new LegacySegmentSpec("1970/3000")).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("tt", "tt")).setVirtualColumns(new ExpressionVirtualColumn("tt", "concat(tags, 'foo')", ColumnType.STRING, TestExprMacroTable.INSTANCE)).setAggregatorSpecs(new CountAggregatorFactory("count")).setContext(context).build();
    Sequence<ResultRow> result = helper.runQueryOnSegmentsObjs(ImmutableList.of(new QueryableIndexSegment(queryableIndex, SegmentId.dummy("sid1")), new IncrementalIndexSegment(incrementalIndex, SegmentId.dummy("sid2"))), query);
    List<ResultRow> expectedResults = Arrays.asList(GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", "foo", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", "t1foo", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", "t2foo", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", "t3foo", "count", 4L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", "t4foo", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", "t5foo", "count", 4L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", "t6foo", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", "t7foo", "count", 2L));
    TestHelper.assertExpectedObjects(expectedResults, result.toList(), "expr-auto");
}
Also used : ResultRow(org.apache.druid.query.groupby.ResultRow) QueryableIndexSegment(org.apache.druid.segment.QueryableIndexSegment) GroupByQuery(org.apache.druid.query.groupby.GroupByQuery) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndexSegment(org.apache.druid.segment.IncrementalIndexSegment) LegacySegmentSpec(org.apache.druid.query.spec.LegacySegmentSpec) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) GroupByQueryRunnerTest(org.apache.druid.query.groupby.GroupByQueryRunnerTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 32 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class MultiValuedDimensionTest method testGroupByExpressionArrayFnArg.

@Test
public void testGroupByExpressionArrayFnArg() {
    if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
        expectedException.expect(RuntimeException.class);
        expectedException.expectMessage("GroupBy v1 does not support dimension selectors with unknown cardinality.");
    }
    GroupByQuery query = GroupByQuery.builder().setDataSource("xx").setQuerySegmentSpec(new LegacySegmentSpec("1970/3000")).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("tt", "tt")).setVirtualColumns(new ExpressionVirtualColumn("tt", "array_to_string(map(tags -> concat('foo', tags), tags), ', ')", ColumnType.STRING, TestExprMacroTable.INSTANCE)).setAggregatorSpecs(new CountAggregatorFactory("count")).setContext(context).build();
    Sequence<ResultRow> result = helper.runQueryOnSegmentsObjs(ImmutableList.of(new QueryableIndexSegment(queryableIndex, SegmentId.dummy("sid1")), new IncrementalIndexSegment(incrementalIndex, SegmentId.dummy("sid2"))), query);
    List<ResultRow> expectedResults = Arrays.asList(GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", NullHandling.replaceWithDefault() ? null : "foo", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", "foot1, foot2, foot3", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", "foot3, foot4, foot5", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "tt", "foot5, foot6, foot7", "count", 2L));
    TestHelper.assertExpectedObjects(expectedResults, result.toList(), "expr-array-fn");
}
Also used : ResultRow(org.apache.druid.query.groupby.ResultRow) QueryableIndexSegment(org.apache.druid.segment.QueryableIndexSegment) GroupByQuery(org.apache.druid.query.groupby.GroupByQuery) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndexSegment(org.apache.druid.segment.IncrementalIndexSegment) LegacySegmentSpec(org.apache.druid.query.spec.LegacySegmentSpec) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) GroupByQueryRunnerTest(org.apache.druid.query.groupby.GroupByQueryRunnerTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 33 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class MultiValuedDimensionTest method testGroupByExpressionMultiConflicting.

@Test
public void testGroupByExpressionMultiConflicting() {
    expectedException.expect(RuntimeException.class);
    expectedException.expectMessage("Invalid expression: (concat [(map ([x] -> (concat [x, othertags])), [tags]), tags]); [tags] used as both scalar and array variables");
    GroupByQuery query = GroupByQuery.builder().setDataSource("xx").setQuerySegmentSpec(new LegacySegmentSpec("1970/3000")).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("texpr", "texpr")).setVirtualColumns(new ExpressionVirtualColumn("texpr", "concat(map((x) -> concat(x, othertags), tags), tags)", ColumnType.STRING, TestExprMacroTable.INSTANCE)).setLimit(5).setAggregatorSpecs(new CountAggregatorFactory("count")).setContext(context).build();
    helper.runQueryOnSegmentsObjs(ImmutableList.of(new QueryableIndexSegment(queryableIndex, SegmentId.dummy("sid1")), new IncrementalIndexSegment(incrementalIndex, SegmentId.dummy("sid2"))), query).toList();
}
Also used : QueryableIndexSegment(org.apache.druid.segment.QueryableIndexSegment) GroupByQuery(org.apache.druid.query.groupby.GroupByQuery) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndexSegment(org.apache.druid.segment.IncrementalIndexSegment) LegacySegmentSpec(org.apache.druid.query.spec.LegacySegmentSpec) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) GroupByQueryRunnerTest(org.apache.druid.query.groupby.GroupByQueryRunnerTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 34 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class TDigestSketchSqlAggregatorTest method testQuantileOnCastedString.

@Test
public void testQuantileOnCastedString() throws Exception {
    cannotVectorize();
    testQuery("SELECT\n" + "  TDIGEST_QUANTILE(CAST(dim1 AS DOUBLE), 0.0),\n" + "  TDIGEST_QUANTILE(CAST(dim1 AS DOUBLE), 0.5),\n" + "  TDIGEST_QUANTILE(CAST(dim1 AS DOUBLE), 1.0)\n" + "FROM foo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(querySegmentSpec(Filtration.eternity())).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "CAST(\"dim1\", 'DOUBLE')", ColumnType.FLOAT, ExprMacroTable.nil())).aggregators(new TDigestSketchAggregatorFactory("a0:agg", "v0", 100)).postAggregators(new TDigestSketchToQuantilePostAggregator("a0", new FieldAccessPostAggregator("a0:agg", "a0:agg"), 0.0), new TDigestSketchToQuantilePostAggregator("a1", new FieldAccessPostAggregator("a0:agg", "a0:agg"), 0.5), new TDigestSketchToQuantilePostAggregator("a2", new FieldAccessPostAggregator("a0:agg", "a0:agg"), 1.0)).context(QUERY_CONTEXT_DEFAULT).build()), ImmutableList.of(NullHandling.replaceWithDefault() ? new Object[] { 0.0, 0.5, 10.1 } : new Object[] { 1.0, 2.0, 10.1 }));
}
Also used : TDigestSketchAggregatorFactory(org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchAggregatorFactory) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) FieldAccessPostAggregator(org.apache.druid.query.aggregation.post.FieldAccessPostAggregator) TDigestSketchToQuantilePostAggregator(org.apache.druid.query.aggregation.tdigestsketch.TDigestSketchToQuantilePostAggregator) BaseCalciteQueryTest(org.apache.druid.sql.calcite.BaseCalciteQueryTest) Test(org.junit.Test)

Example 35 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class DoublesSketchSqlAggregatorTest method testQuantileOnCastedString.

@Test
public void testQuantileOnCastedString() throws Exception {
    final List<Object[]> expectedResults;
    if (NullHandling.replaceWithDefault()) {
        expectedResults = ImmutableList.of(new Object[] { 0.0, 1.0, 10.1, 10.1, 20.2, 0.0, 10.1, 0.0 });
    } else {
        expectedResults = ImmutableList.of(new Object[] { 1.0, 2.0, 10.1, 10.1, 20.2, Double.NaN, 10.1, Double.NaN });
    }
    testQuery("SELECT\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.01),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.5, 64),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.98, 256),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.99),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE) * 2, 0.97),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.99) FILTER(WHERE dim2 = 'abc'),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.999) FILTER(WHERE dim2 <> 'abc'),\n" + "APPROX_QUANTILE_DS(CAST(dim1 as DOUBLE), 0.999) FILTER(WHERE dim2 = 'abc')\n" + "FROM foo", ImmutableList.of(Druids.newTimeseriesQueryBuilder().dataSource(CalciteTests.DATASOURCE1).intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Filtration.eternity()))).granularity(Granularities.ALL).virtualColumns(new ExpressionVirtualColumn("v0", "CAST(\"dim1\", 'DOUBLE')", ColumnType.FLOAT, TestExprMacroTable.INSTANCE), new ExpressionVirtualColumn("v1", "(CAST(\"dim1\", 'DOUBLE') * 2)", ColumnType.FLOAT, TestExprMacroTable.INSTANCE)).aggregators(ImmutableList.of(new DoublesSketchAggregatorFactory("a0:agg", "v0", 128), new DoublesSketchAggregatorFactory("a1:agg", "v0", 64), new DoublesSketchAggregatorFactory("a2:agg", "v0", 256), new DoublesSketchAggregatorFactory("a4:agg", "v1", 128), new FilteredAggregatorFactory(new DoublesSketchAggregatorFactory("a5:agg", "v0", 128), new SelectorDimFilter("dim2", "abc", null)), new FilteredAggregatorFactory(new DoublesSketchAggregatorFactory("a6:agg", "v0", 128), new NotDimFilter(new SelectorDimFilter("dim2", "abc", null))))).postAggregators(new DoublesSketchToQuantilePostAggregator("a0", makeFieldAccessPostAgg("a0:agg"), 0.01f), new DoublesSketchToQuantilePostAggregator("a1", makeFieldAccessPostAgg("a1:agg"), 0.50f), new DoublesSketchToQuantilePostAggregator("a2", makeFieldAccessPostAgg("a2:agg"), 0.98f), new DoublesSketchToQuantilePostAggregator("a3", makeFieldAccessPostAgg("a0:agg"), 0.99f), new DoublesSketchToQuantilePostAggregator("a4", makeFieldAccessPostAgg("a4:agg"), 0.97f), new DoublesSketchToQuantilePostAggregator("a5", makeFieldAccessPostAgg("a5:agg"), 0.99f), new DoublesSketchToQuantilePostAggregator("a6", makeFieldAccessPostAgg("a6:agg"), 0.999f), new DoublesSketchToQuantilePostAggregator("a7", makeFieldAccessPostAgg("a5:agg"), 0.999f)).context(QUERY_CONTEXT_DEFAULT).build()), expectedResults);
}
Also used : FilteredAggregatorFactory(org.apache.druid.query.aggregation.FilteredAggregatorFactory) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) NotDimFilter(org.apache.druid.query.filter.NotDimFilter) SelectorDimFilter(org.apache.druid.query.filter.SelectorDimFilter) DoublesSketchToQuantilePostAggregator(org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchToQuantilePostAggregator) MultipleIntervalSegmentSpec(org.apache.druid.query.spec.MultipleIntervalSegmentSpec) DoublesSketchAggregatorFactory(org.apache.druid.query.aggregation.datasketches.quantiles.DoublesSketchAggregatorFactory) BaseCalciteQueryTest(org.apache.druid.sql.calcite.BaseCalciteQueryTest) Test(org.junit.Test)

Aggregations

ExpressionVirtualColumn (org.apache.druid.segment.virtual.ExpressionVirtualColumn)86 Test (org.junit.Test)73 DefaultDimensionSpec (org.apache.druid.query.dimension.DefaultDimensionSpec)58 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)45 LongSumAggregatorFactory (org.apache.druid.query.aggregation.LongSumAggregatorFactory)26 CountAggregatorFactory (org.apache.druid.query.aggregation.CountAggregatorFactory)24 QueryableIndexSegment (org.apache.druid.segment.QueryableIndexSegment)18 GroupByQueryRunnerTest (org.apache.druid.query.groupby.GroupByQueryRunnerTest)17 GroupByQuery (org.apache.druid.query.groupby.GroupByQuery)16 LegacySegmentSpec (org.apache.druid.query.spec.LegacySegmentSpec)15 IncrementalIndexSegment (org.apache.druid.segment.IncrementalIndexSegment)15 ResultRow (org.apache.druid.query.groupby.ResultRow)13 MultipleIntervalSegmentSpec (org.apache.druid.query.spec.MultipleIntervalSegmentSpec)13 BaseCalciteQueryTest (org.apache.druid.sql.calcite.BaseCalciteQueryTest)11 VirtualColumns (org.apache.druid.segment.VirtualColumns)10 ImmutableList (com.google.common.collect.ImmutableList)8 TimeUnit (java.util.concurrent.TimeUnit)8 NullHandling (org.apache.druid.common.config.NullHandling)8 Closer (org.apache.druid.java.util.common.io.Closer)8 QueryDataSource (org.apache.druid.query.QueryDataSource)8