Search in sources :

Example 21 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class JoinFilterAnalyzerTest method test_filterPushDown_factConcatExpressionToCountryLeftFilterOnChannelAndCountryName.

@Test
public void test_filterPushDown_factConcatExpressionToCountryLeftFilterOnChannelAndCountryName() {
    JoinableClause factExprToCountry = new JoinableClause(FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX, new IndexedTableJoinable(countriesTable), JoinType.LEFT, JoinConditionAnalysis.forExpression(StringUtils.format("\"%scountryIsoCode\" == concat(countryIsoCode, regionIsoCode)", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX), FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX, ExprMacroTable.nil()));
    List<JoinableClause> joinableClauses = ImmutableList.of(factExprToCountry);
    Filter filter = new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#en.wikipedia"), new SelectorFilter("c1.countryName", "Usca")));
    JoinFilterPreAnalysis joinFilterPreAnalysis = makeDefaultConfigPreAnalysis(filter, joinableClauses, VirtualColumns.EMPTY);
    HashJoinSegmentStorageAdapter adapter = new HashJoinSegmentStorageAdapter(factSegment.asStorageAdapter(), joinableClauses, joinFilterPreAnalysis);
    JoinTestHelper.verifyCursors(adapter.makeCursors(filter, Intervals.ETERNITY, VirtualColumns.EMPTY, Granularities.ALL, false, null), ImmutableList.of("page", FACT_TO_COUNTRY_ON_ISO_CODE_PREFIX + "countryName"), ImmutableList.of(new Object[] { "President of India", "Usca" }, new Object[] { "Otjiwarongo Airport", "Usca" }, new Object[] { "Carlo Curti", "Usca" }));
    ExpressionVirtualColumn expectedVirtualColumn = new ExpressionVirtualColumn("JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-0", "concat(countryIsoCode, regionIsoCode)", ColumnType.STRING, ExprMacroTable.nil());
    JoinFilterSplit expectedFilterSplit = new JoinFilterSplit(new AndFilter(ImmutableList.of(new SelectorFilter("channel", "#en.wikipedia"), new InDimFilter("JOIN-FILTER-PUSHDOWN-VIRTUAL-COLUMN-0", ImmutableSet.of("USCA"), null, null).toFilter())), new SelectorFilter("c1.countryName", "Usca"), ImmutableSet.of(expectedVirtualColumn));
    JoinFilterSplit actualFilterSplit = JoinFilterAnalyzer.splitFilter(joinFilterPreAnalysis);
    Assert.assertEquals(expectedFilterSplit.getBaseTableFilter(), actualFilterSplit.getBaseTableFilter());
    Assert.assertEquals(expectedFilterSplit.getJoinTableFilter(), actualFilterSplit.getJoinTableFilter());
    ExpressionVirtualColumn actualVirtualColumn = (ExpressionVirtualColumn) actualFilterSplit.getPushDownVirtualColumns().iterator().next();
    compareExpressionVirtualColumns(expectedVirtualColumn, actualVirtualColumn);
}
Also used : AndFilter(org.apache.druid.segment.filter.AndFilter) SelectorFilter(org.apache.druid.segment.filter.SelectorFilter) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) JoinFilterPreAnalysis(org.apache.druid.segment.join.filter.JoinFilterPreAnalysis) FalseFilter(org.apache.druid.segment.filter.FalseFilter) OrFilter(org.apache.druid.segment.filter.OrFilter) BoundDimFilter(org.apache.druid.query.filter.BoundDimFilter) SelectorFilter(org.apache.druid.segment.filter.SelectorFilter) AndFilter(org.apache.druid.segment.filter.AndFilter) BoundFilter(org.apache.druid.segment.filter.BoundFilter) InDimFilter(org.apache.druid.query.filter.InDimFilter) ExpressionDimFilter(org.apache.druid.query.filter.ExpressionDimFilter) Filter(org.apache.druid.query.filter.Filter) JoinFilterSplit(org.apache.druid.segment.join.filter.JoinFilterSplit) IndexedTableJoinable(org.apache.druid.segment.join.table.IndexedTableJoinable) InDimFilter(org.apache.druid.query.filter.InDimFilter) Test(org.junit.Test)

Example 22 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class RowBasedStorageAdapterTest method test_makeCursors_filterOnVirtualColumn.

@Test
public void test_makeCursors_filterOnVirtualColumn() {
    final RowBasedStorageAdapter<Integer> adapter = createIntAdapter(0, 1);
    final Sequence<Cursor> cursors = adapter.makeCursors(new SelectorDimFilter("vc", "2", null).toFilter(), Intervals.ETERNITY, VirtualColumns.create(ImmutableList.of(new ExpressionVirtualColumn("vc", "\"LONG\" + 1", ColumnType.LONG, ExprMacroTable.nil()))), Granularities.ALL, false, null);
    Assert.assertEquals(ImmutableList.of(ImmutableList.of("1")), walkCursors(cursors, READ_STRING));
    Assert.assertEquals(1, numCloses.get());
}
Also used : ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) SelectorDimFilter(org.apache.druid.query.filter.SelectorDimFilter) Test(org.junit.Test)

Example 23 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class MultiValuedDimensionTest method testGroupByExpressionMultiMultiAutoAutoDupeIdentifier.

@Test
public void testGroupByExpressionMultiMultiAutoAutoDupeIdentifier() {
    GroupByQuery query = GroupByQuery.builder().setDataSource("xx").setQuerySegmentSpec(new LegacySegmentSpec("1970/3000")).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("texpr", "texpr")).setVirtualColumns(new ExpressionVirtualColumn("texpr", "concat(tags, tags)", ColumnType.STRING, TestExprMacroTable.INSTANCE)).setLimitSpec(new DefaultLimitSpec(ImmutableList.of(new OrderByColumnSpec("count", OrderByColumnSpec.Direction.DESCENDING)), 5)).setAggregatorSpecs(new CountAggregatorFactory("count")).setContext(context).build();
    Sequence<ResultRow> result = helper.runQueryOnSegmentsObjs(ImmutableList.of(new QueryableIndexSegment(queryableIndex, SegmentId.dummy("sid1")), new IncrementalIndexSegment(incrementalIndex, SegmentId.dummy("sid2"))), query);
    List<ResultRow> expectedResults = Arrays.asList(GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t3t3", "count", 4L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t5t5", "count", 4L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", NullHandling.emptyToNullIfNeeded(""), "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t1t1", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t2t2", "count", 2L));
    TestHelper.assertExpectedObjects(expectedResults, result.toList(), "expr-multi-multi-auto-auto-self");
}
Also used : OrderByColumnSpec(org.apache.druid.query.groupby.orderby.OrderByColumnSpec) ResultRow(org.apache.druid.query.groupby.ResultRow) QueryableIndexSegment(org.apache.druid.segment.QueryableIndexSegment) GroupByQuery(org.apache.druid.query.groupby.GroupByQuery) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) DefaultLimitSpec(org.apache.druid.query.groupby.orderby.DefaultLimitSpec) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndexSegment(org.apache.druid.segment.IncrementalIndexSegment) LegacySegmentSpec(org.apache.druid.query.spec.LegacySegmentSpec) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) GroupByQueryRunnerTest(org.apache.druid.query.groupby.GroupByQueryRunnerTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 24 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class MultiValuedDimensionTest method testGroupByExpression.

@Test
public void testGroupByExpression() {
    if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
        expectedException.expect(RuntimeException.class);
        expectedException.expectMessage("GroupBy v1 does not support dimension selectors with unknown cardinality.");
    }
    GroupByQuery query = GroupByQuery.builder().setDataSource("xx").setQuerySegmentSpec(new LegacySegmentSpec("1970/3000")).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("texpr", "texpr")).setVirtualColumns(new ExpressionVirtualColumn("texpr", "map(x -> concat(x, 'foo'), tags)", ColumnType.STRING, TestExprMacroTable.INSTANCE)).setAggregatorSpecs(new CountAggregatorFactory("count")).setContext(context).build();
    Sequence<ResultRow> result = helper.runQueryOnSegmentsObjs(ImmutableList.of(new QueryableIndexSegment(queryableIndex, SegmentId.dummy("sid1")), new IncrementalIndexSegment(incrementalIndex, SegmentId.dummy("sid2"))), query);
    List<ResultRow> expectedResults = Arrays.asList(GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", NullHandling.sqlCompatible() ? "foo" : null, "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t1foo", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t2foo", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t3foo", "count", 4L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t4foo", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t5foo", "count", 4L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t6foo", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t7foo", "count", 2L));
    TestHelper.assertExpectedObjects(expectedResults, result.toList(), "expr");
}
Also used : ResultRow(org.apache.druid.query.groupby.ResultRow) QueryableIndexSegment(org.apache.druid.segment.QueryableIndexSegment) GroupByQuery(org.apache.druid.query.groupby.GroupByQuery) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndexSegment(org.apache.druid.segment.IncrementalIndexSegment) LegacySegmentSpec(org.apache.druid.query.spec.LegacySegmentSpec) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) GroupByQueryRunnerTest(org.apache.druid.query.groupby.GroupByQueryRunnerTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Example 25 with ExpressionVirtualColumn

use of org.apache.druid.segment.virtual.ExpressionVirtualColumn in project druid by druid-io.

the class MultiValuedDimensionTest method testGroupByExpressionMultiMulti.

@Test
public void testGroupByExpressionMultiMulti() {
    if (config.getDefaultStrategy().equals(GroupByStrategySelector.STRATEGY_V1)) {
        expectedException.expect(RuntimeException.class);
        expectedException.expectMessage("GroupBy v1 does not support dimension selectors with unknown cardinality.");
    }
    GroupByQuery query = GroupByQuery.builder().setDataSource("xx").setQuerySegmentSpec(new LegacySegmentSpec("1970/3000")).setGranularity(Granularities.ALL).setDimensions(new DefaultDimensionSpec("texpr", "texpr")).setVirtualColumns(new ExpressionVirtualColumn("texpr", "cartesian_map((x,y) -> concat(x, y), tags, othertags)", ColumnType.STRING, TestExprMacroTable.INSTANCE)).setLimit(5).setAggregatorSpecs(new CountAggregatorFactory("count")).setContext(context).build();
    Sequence<ResultRow> result = helper.runQueryOnSegmentsObjs(ImmutableList.of(new QueryableIndexSegment(queryableIndex, SegmentId.dummy("sid1")), new IncrementalIndexSegment(incrementalIndex, SegmentId.dummy("sid2"))), query);
    List<ResultRow> expectedResults = NullHandling.sqlCompatible() ? Arrays.asList(GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t1u1", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t1u2", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t2u1", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t2u2", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t3u1", "count", 2L)) : Arrays.asList(GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", null, "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t1u1", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t1u2", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t2u1", "count", 2L), GroupByQueryRunnerTestHelper.createExpectedRow(query, "1970", "texpr", "t2u2", "count", 2L));
    TestHelper.assertExpectedObjects(expectedResults, result.toList(), "expr-multi-multi");
}
Also used : ResultRow(org.apache.druid.query.groupby.ResultRow) QueryableIndexSegment(org.apache.druid.segment.QueryableIndexSegment) GroupByQuery(org.apache.druid.query.groupby.GroupByQuery) ExpressionVirtualColumn(org.apache.druid.segment.virtual.ExpressionVirtualColumn) CountAggregatorFactory(org.apache.druid.query.aggregation.CountAggregatorFactory) IncrementalIndexSegment(org.apache.druid.segment.IncrementalIndexSegment) LegacySegmentSpec(org.apache.druid.query.spec.LegacySegmentSpec) DefaultDimensionSpec(org.apache.druid.query.dimension.DefaultDimensionSpec) GroupByQueryRunnerTest(org.apache.druid.query.groupby.GroupByQueryRunnerTest) InitializedNullHandlingTest(org.apache.druid.testing.InitializedNullHandlingTest) Test(org.junit.Test)

Aggregations

ExpressionVirtualColumn (org.apache.druid.segment.virtual.ExpressionVirtualColumn)86 Test (org.junit.Test)73 DefaultDimensionSpec (org.apache.druid.query.dimension.DefaultDimensionSpec)58 InitializedNullHandlingTest (org.apache.druid.testing.InitializedNullHandlingTest)45 LongSumAggregatorFactory (org.apache.druid.query.aggregation.LongSumAggregatorFactory)26 CountAggregatorFactory (org.apache.druid.query.aggregation.CountAggregatorFactory)24 QueryableIndexSegment (org.apache.druid.segment.QueryableIndexSegment)18 GroupByQueryRunnerTest (org.apache.druid.query.groupby.GroupByQueryRunnerTest)17 GroupByQuery (org.apache.druid.query.groupby.GroupByQuery)16 LegacySegmentSpec (org.apache.druid.query.spec.LegacySegmentSpec)15 IncrementalIndexSegment (org.apache.druid.segment.IncrementalIndexSegment)15 ResultRow (org.apache.druid.query.groupby.ResultRow)13 MultipleIntervalSegmentSpec (org.apache.druid.query.spec.MultipleIntervalSegmentSpec)13 BaseCalciteQueryTest (org.apache.druid.sql.calcite.BaseCalciteQueryTest)11 VirtualColumns (org.apache.druid.segment.VirtualColumns)10 ImmutableList (com.google.common.collect.ImmutableList)8 TimeUnit (java.util.concurrent.TimeUnit)8 NullHandling (org.apache.druid.common.config.NullHandling)8 Closer (org.apache.druid.java.util.common.io.Closer)8 QueryDataSource (org.apache.druid.query.QueryDataSource)8